-------------------------------------------------------------------------------
-- (C) Altran Praxis Limited
-------------------------------------------------------------------------------
--
-- The SPARK toolset is free software; you can redistribute it and/or modify it
-- under terms of the GNU General Public License as published by the Free
-- Software Foundation; either version 3, or (at your option) any later
-- version. The SPARK toolset is distributed in the hope that it will be
-- useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
-- Public License for more details. You should have received a copy of the GNU
-- General Public License distributed with the SPARK toolset; see file
-- COPYING3. If not, go to http://www.gnu.org/licenses for a complete copy of
-- the license.
--
--=============================================================================

with LexTokenLists;

separate (SparkLex)
procedure Lex
  (Prog_Text    : in     SPARK_IO.File_Type;
   Allow_Dollar : in     Boolean;
   Curr_Line    : in out Line_Context;
   Token        :    out SP_Symbols.SP_Terminal;
   Lex_Val      :    out LexTokenManager.Lex_Value;
   Punct_Token  :    out Boolean)
is
   Start_Line                      : LexTokenManager.Line_Numbers;
   Start_Posn                      : E_Strings.Positions;
   End_Posn                        : E_Strings.Lengths;
   Lex_Str                         : LexTokenManager.Lex_String;
   Next_Token                      : SP_Symbols.SP_Terminal;
   Ch                              : Character;
   Hidden_Part, End_Hide_Found     : Boolean;
   Hide_Designator, End_Designator : LexTokenLists.Lists;
   Previous_Context                : Program_Context;

   procedure GetIdent (Curr_Line : in out Line_Context;
                       Token     :    out SP_Symbols.SP_Terminal)
   --# derives Curr_Line,
   --#         Token     from Curr_Line;
   --# pre E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) and
   --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
   --# post E_Strings.Get_Length (Curr_Line.Conts) = E_Strings.Get_Length (Curr_Line~.Conts) and
   --#   Curr_Line.Curr_Pos > Curr_Line~.Curr_Pos and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
   --#   Curr_Line.Last_Token_Pos = Curr_Line~.Last_Token_Pos and
   --#   Token /= SP_Symbols.annotation_end and Token /= SP_Symbols.SPEND;
      is separate;

   procedure GetNumber (Curr_Line : in out Line_Context;
                        Token     :    out SP_Symbols.SP_Terminal)
   --# derives Curr_Line,
   --#         Token     from Curr_Line;
   --# pre E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) and
   --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
   --# post E_Strings.Get_Length (Curr_Line.Conts) = E_Strings.Get_Length (Curr_Line~.Conts) and
   --#   Curr_Line.Curr_Pos > Curr_Line~.Curr_Pos and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
   --#   Curr_Line.Last_Token_Pos = Curr_Line~.Last_Token_Pos and
   --#   Token /= SP_Symbols.annotation_end and Token /= SP_Symbols.SPEND;
      is separate;

   procedure HyphIntro (Curr_Line : in out Line_Context;
                        Token     :    out SP_Symbols.SP_Terminal)
   --# global in     CommandLineData.Content;
   --#        in     Dictionary.Dict;
   --#        in     LexTokenManager.State;
   --#        in out ErrorHandler.Error_Context;
   --#        in out SPARK_IO.File_Sys;
   --# derives Curr_Line,
   --#         Token                      from CommandLineData.Content,
   --#                                         Curr_Line &
   --#         ErrorHandler.Error_Context,
   --#         SPARK_IO.File_Sys          from CommandLineData.Content,
   --#                                         Curr_Line,
   --#                                         Dictionary.Dict,
   --#                                         ErrorHandler.Error_Context,
   --#                                         LexTokenManager.State,
   --#                                         SPARK_IO.File_Sys;
   --# pre E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) and
   --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
   --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
   --# post E_Strings.Get_Length (Curr_Line.Conts) = E_Strings.Get_Length (Curr_Line~.Conts) and
   --#   Curr_Line.Curr_Pos >= Curr_Line~.Curr_Pos and
   --#   ((Token /= SP_Symbols.annotation_end and Token /= SP_Symbols.SPEND) <-> (Curr_Line.Curr_Pos > Curr_Line~.Curr_Pos)) and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
   --#   Curr_Line.Last_Token_Pos = Curr_Line~.Last_Token_Pos;
      is separate;

   procedure GetString (Curr_Line : in out Line_Context;
                        Token     :    out SP_Symbols.SP_Terminal)
   --# derives Curr_Line,
   --#         Token     from Curr_Line;
   --# pre E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) and
   --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
   --# post E_Strings.Get_Length (Curr_Line.Conts) = E_Strings.Get_Length (Curr_Line~.Conts) and
   --#   Curr_Line.Curr_Pos > Curr_Line~.Curr_Pos and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
   --#   Curr_Line.Last_Token_Pos = Curr_Line~.Last_Token_Pos and
   --#   Token /= SP_Symbols.annotation_end and Token /= SP_Symbols.SPEND;
      is separate;

   procedure ApostIntro (Curr_Line : in out Line_Context;
                         Token     :    out SP_Symbols.SP_Terminal)
   --# derives Curr_Line,
   --#         Token     from Curr_Line;
   --# pre E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) and
   --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
   --# post E_Strings.Get_Length (Curr_Line.Conts) = E_Strings.Get_Length (Curr_Line~.Conts) and
   --#   Curr_Line.Curr_Pos > Curr_Line~.Curr_Pos and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
   --#   Curr_Line.Last_Token_Pos = Curr_Line~.Last_Token_Pos and
   --#   Token /= SP_Symbols.annotation_end and Token /= SP_Symbols.SPEND;
      is separate;

   procedure LTIntro (Curr_Line : in out Line_Context;
                      Token     :    out SP_Symbols.SP_Terminal)
   --# derives Curr_Line,
   --#         Token     from Curr_Line;
   --# pre E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) and
   --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
   --# post E_Strings.Get_Length (Curr_Line.Conts) = E_Strings.Get_Length (Curr_Line~.Conts) and
   --#   Curr_Line.Curr_Pos > Curr_Line~.Curr_Pos and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
   --#   Curr_Line.Last_Token_Pos = Curr_Line~.Last_Token_Pos and
   --#   Token /= SP_Symbols.annotation_end and Token /= SP_Symbols.SPEND;
      is separate;

   procedure NextLex (Curr_Line : in out Line_Context;
                      Token     :    out SP_Symbols.SP_Terminal)
   --# global in     CommandLineData.Content;
   --#        in     Dictionary.Dict;
   --#        in     LexTokenManager.State;
   --#        in out ErrorHandler.Error_Context;
   --#        in out SPARK_IO.File_Sys;
   --# derives Curr_Line,
   --#         Token                      from CommandLineData.Content,
   --#                                         Curr_Line &
   --#         ErrorHandler.Error_Context,
   --#         SPARK_IO.File_Sys          from CommandLineData.Content,
   --#                                         Curr_Line,
   --#                                         Dictionary.Dict,
   --#                                         ErrorHandler.Error_Context,
   --#                                         LexTokenManager.State,
   --#                                         SPARK_IO.File_Sys;
   --# pre E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) and
   --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
   --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
   --# post E_Strings.Get_Length (Curr_Line.Conts) = E_Strings.Get_Length (Curr_Line~.Conts) and
   --#   Curr_Line.Curr_Pos >= Curr_Line~.Curr_Pos and
   --#   ((Token /= SP_Symbols.annotation_end and Token /= SP_Symbols.SPEND) <-> (Curr_Line.Curr_Pos > Curr_Line~.Curr_Pos)) and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
   --#   Curr_Line.Last_Token_Pos = Curr_Line~.Curr_Pos;
   --
   -- The Ada Lexis allows the class of a token to be determined by the
   -- first character in the text string representing it. Given the first
   -- character of the string representing the token is at the current position
   -- in the line buffer, Curr_Line, Next_Lex determines the class of the token and
   -- calls a procedure to recognise that class of token.
   -- On exit Token is set to a value representing the token.
      is separate;

   -- Modified list of reserved words which are punctuation tokens
   -- this is a knock on effect from change in syntax of proof contexts
   function Is_Punct_Token (Token : SP_Symbols.SP_Terminal) return Boolean is
      Result : Boolean;
   begin
      case Token is
         when SP_Symbols.left_paren       |
           SP_Symbols.right_paren      |
           SP_Symbols.comma            |
           SP_Symbols.colon            |
           SP_Symbols.semicolon        |
           SP_Symbols.becomes          |
           SP_Symbols.double_dot       |
           SP_Symbols.point            |
           SP_Symbols.apostrophe       |
           SP_Symbols.vertical_bar     |
           SP_Symbols.arrow            |
           SP_Symbols.annotation_start |
           SP_Symbols.annotation_end   |
           SP_Symbols.proof_context    |
           SP_Symbols.RWabort          |
           SP_Symbols.RWaccess         |
           SP_Symbols.RWall            |
           SP_Symbols.RWarray          |
           SP_Symbols.RWassert         |
           SP_Symbols.RWassume         |
           SP_Symbols.RWat             |
           SP_Symbols.RWbegin          |
           SP_Symbols.RWbody           |
           SP_Symbols.RWcase           |
           SP_Symbols.RWcheck          |
           SP_Symbols.RWconstant       |
           SP_Symbols.RWdeclare        |
           SP_Symbols.RWdelta          |
           SP_Symbols.RWderives        |
           SP_Symbols.RWdigits         |
           SP_Symbols.RWdo             |
           SP_Symbols.RWelse           |
           SP_Symbols.RWelsif          |
           SP_Symbols.RWend            |
           SP_Symbols.RWentry          |
           SP_Symbols.RWexception      |
           SP_Symbols.RWfor            |
           SP_Symbols.RWfrom           |
           SP_Symbols.RWfunction       |
           SP_Symbols.RWgeneric        |
           SP_Symbols.RWglobal         |
           SP_Symbols.RWgoto           |
           SP_Symbols.RWhide           |
           SP_Symbols.RWif             |
           SP_Symbols.RWin             |
           SP_Symbols.RWinherit        |
           SP_Symbols.RWinitializes    |
           SP_Symbols.RWis             |
           SP_Symbols.RWlimited        |
           SP_Symbols.RWloop           |
           SP_Symbols.RWnew            |
           SP_Symbols.RWnotin          |
           SP_Symbols.RWof             |
           SP_Symbols.RWothers         |
           SP_Symbols.RWout            |
           SP_Symbols.RWown            |
           SP_Symbols.RWpackage        |
           SP_Symbols.RWpost           |
           SP_Symbols.RWpragma         |
           SP_Symbols.RWpre            |
           SP_Symbols.RWprivate        |
           SP_Symbols.RWprocedure      |
           SP_Symbols.RWraise          |
           SP_Symbols.RWrange          |
           SP_Symbols.RWrecord         |
           SP_Symbols.RWrenames        |
           SP_Symbols.RWreturn         |
           SP_Symbols.RWreverse        |
           SP_Symbols.RWselect         |
           SP_Symbols.RWseparate       |
           SP_Symbols.RWsubtype        |
           SP_Symbols.RWtask           |
           SP_Symbols.RWterminate      |
           SP_Symbols.RWthen           |
           SP_Symbols.RWtype           |
           SP_Symbols.RWuse            |
           SP_Symbols.RWwhen           |
           SP_Symbols.RWwhile          |
           SP_Symbols.RWwith           |
           -- SPARK95 reserved words
           SP_Symbols.RWabstract       |
           SP_Symbols.RWrequeue        |
           SP_Symbols.RWprotected      |
           SP_Symbols.RWuntil          |
           -- SPARK2005 reserved words
           SP_Symbols.RWinterface      |
           SP_Symbols.RWsynchronized   =>
            Result := True;
         when others =>
            Result := False;
      end case;
      return Result;
   end Is_Punct_Token;

   procedure Check_Following_Token
     (Prog_Text         : in     SPARK_IO.File_Type;
      Possible_Follower : in     SP_Symbols.SP_Symbol;
      Transformed_Token : in     SP_Symbols.SP_Terminal;
      Curr_Line         : in out Line_Context;
      Next_Token        : in out SP_Symbols.SP_Terminal;
      Lex_Str           :    out LexTokenManager.Lex_String)
   --# global in     CommandLineData.Content;
   --#        in     Dictionary.Dict;
   --#        in out ErrorHandler.Error_Context;
   --#        in out LexTokenManager.State;
   --#        in out SPARK_IO.File_Sys;
   --# derives Curr_Line                  from *,
   --#                                         CommandLineData.Content,
   --#                                         Dictionary.Dict,
   --#                                         ErrorHandler.Error_Context,
   --#                                         LexTokenManager.State,
   --#                                         Possible_Follower,
   --#                                         Prog_Text,
   --#                                         SPARK_IO.File_Sys &
   --#         ErrorHandler.Error_Context,
   --#         LexTokenManager.State,
   --#         SPARK_IO.File_Sys          from CommandLineData.Content,
   --#                                         Curr_Line,
   --#                                         Dictionary.Dict,
   --#                                         ErrorHandler.Error_Context,
   --#                                         LexTokenManager.State,
   --#                                         Prog_Text,
   --#                                         SPARK_IO.File_Sys &
   --#         Lex_Str                    from  &
   --#         Next_Token                 from *,
   --#                                         CommandLineData.Content,
   --#                                         Curr_Line,
   --#                                         Dictionary.Dict,
   --#                                         ErrorHandler.Error_Context,
   --#                                         LexTokenManager.State,
   --#                                         Possible_Follower,
   --#                                         Prog_Text,
   --#                                         SPARK_IO.File_Sys,
   --#                                         Transformed_Token;
   --# pre E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
   --# post E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts)  + 1;
   is
      Following_Token : SP_Symbols.SP_Terminal;
   begin
      LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                 Curr_Line => Curr_Line);
      LineManager.Record_Curr_Pos (Curr_Line => Curr_Line);
      GetIdent (Curr_Line => Curr_Line,
                Token     => Following_Token);
      if Following_Token = SP_Symbols.identifier then
         Check_Reserved
           (Curr_Line  => Curr_Line,
            Start_Pos  => Curr_Line.Last_Token_Pos,
            End_Pos    => Curr_Line.Curr_Pos - 1,
            Look_Ahead => True,
            Token      => Following_Token);
         if Following_Token = Possible_Follower then
            Next_Token := Transformed_Token;
         else
            LineManager.Reset_Curr_Pos (Curr_Line => Curr_Line);
         end if;
      else
         LineManager.Reset_Curr_Pos (Curr_Line => Curr_Line);
      end if;
      Lex_Str := LexTokenManager.Null_String;
   end Check_Following_Token;

   ---------------------

   procedure Insert_String_Literal
     (Line_In            : in     E_Strings.T;
      Start_Pos, End_Pos : in     E_Strings.Positions;
      Lex_Str            :    out LexTokenManager.Lex_String)
   --# global in out LexTokenManager.State;
   --# derives LexTokenManager.State,
   --#         Lex_Str               from End_Pos,
   --#                                    LexTokenManager.State,
   --#                                    Line_In,
   --#                                    Start_Pos;
   --# pre Start_Pos <= E_Strings.Get_Length (Line_In) and
   --#   End_Pos <= E_Strings.Get_Length (Line_In);
   is
      Modified_Line : E_Strings.T := E_Strings.Empty_String;
      Ch            : Character;
      Skip_Next     : Boolean     := False;
   begin
      if Start_Pos < End_Pos then
         -- copy leading quote
         E_Strings.Append_Char (E_Str => Modified_Line,
                                Ch    => E_Strings.Get_Element (E_Str => Line_In,
                                                                Pos   => Start_Pos));

         -- copy character up to closing quote eliminating doubled quotes
         for I in E_Strings.Positions range Start_Pos + 1 .. End_Pos - 1 loop
            if Skip_Next then
               Skip_Next := False;
            else
               Ch := E_Strings.Get_Element (E_Str => Line_In,
                                            Pos   => I);
               E_Strings.Append_Char (E_Str => Modified_Line,
                                      Ch    => Ch);
               Skip_Next := Ch = '"';
            end if;
         end loop;

         -- copy closing quote
         E_Strings.Append_Char (E_Str => Modified_Line,
                                Ch    => E_Strings.Get_Element (E_Str => Line_In,
                                                                Pos   => End_Pos));
         -- put in string table
         LexTokenManager.Insert_Examiner_String (Str     => Modified_Line,
                                                 Lex_Str => Lex_Str);
      else
         Lex_Str := LexTokenManager.Null_String;
      end if;
   end Insert_String_Literal;

   ---------------------

   procedure Insert_Lex_String
     (The_Line           : in     E_Strings.T;
      Start_Pos, End_Pos : in     E_Strings.Positions;
      Lex_Str            :    out LexTokenManager.Lex_String)
   --# global in out LexTokenManager.State;
   --# derives LexTokenManager.State,
   --#         Lex_Str               from End_Pos,
   --#                                    LexTokenManager.State,
   --#                                    Start_Pos,
   --#                                    The_Line;
   is
      E_Str : E_Strings.T := E_Strings.Empty_String;
   begin
      for I in E_Strings.Positions range Start_Pos .. End_Pos loop
         E_Strings.Append_Char (E_Str => E_Str,
                                Ch    => E_Strings.Get_Element (E_Str => The_Line,
                                                                Pos   => I));
      end loop;
      LexTokenManager.Insert_Examiner_String (Str     => E_Str,
                                              Lex_Str => Lex_Str);
   end Insert_Lex_String;

begin -- Lex

   Hidden_Part     := False;
   Hide_Designator := LexTokenLists.Null_List;
   loop
      --# assert E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
      --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
      --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
      --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
      Previous_Context := Curr_Line.Context;
      LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                 Curr_Line => Curr_Line);
      Start_Line := Curr_Line.Line_No;
      Start_Posn := Curr_Line.Curr_Pos;
      if Previous_Context = In_Annotation and then Curr_Line.Context = In_Ada then
         Next_Token := SP_Symbols.annotation_end;
      else
         NextLex (Curr_Line => Curr_Line,
                  Token     => Next_Token);
      end if;
      End_Posn := Curr_Line.Curr_Pos - 1;

      --# assert E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
      --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
      --#   Curr_Line.Lookahead_Pos = Curr_Line.Curr_Pos and
      --#   Curr_Line.Last_Token_Pos <= Curr_Line.Curr_Pos and
      --#   Start_Posn <= E_Strings.Get_Length (Curr_Line.Conts) and
      --#   End_Posn <= E_Strings.Get_Length (Curr_Line.Conts) and
      --#   ((Next_Token /= SP_Symbols.annotation_end and Next_Token /= SP_Symbols.SPEND) <-> (Start_Posn <= End_Posn));

      case Next_Token is
         when SP_Symbols.identifier =>
            Check_Reserved
              (Curr_Line  => Curr_Line,
               Start_Pos  => Start_Posn,
               End_Pos    => End_Posn,
               Look_Ahead => False,
               Token      => Next_Token);
            if Next_Token = SP_Symbols.identifier then

               if E_Strings.Get_Element (E_Str => Curr_Line.Conts,
                                         Pos   => Start_Posn) = '$' then
                  --# accept F, 41, "Stable expression here OK";
                  if Allow_Dollar then
                     null;
                  else
                     -- Identifier starting with a $, which is illegal in
                     -- this case.  Issue a lexical error and then treat
                     -- as an identifier starting at the next character.
                     ErrorHandler.Lex_Error
                       (Error_Message    => "Illegal token",
                        Recovery_Message => "Token ignored",
                        Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                                    Start_Pos     => Start_Posn),
                                                                       Token_Str => LexTokenManager.Null_String));
                     Start_Posn := Start_Posn + 1;
                  end if;
                  --# end accept;
               end if;
               Insert_Lex_String (The_Line  => Curr_Line.Conts,
                                  Start_Pos => Start_Posn,
                                  End_Pos   => End_Posn,
                                  Lex_Str   => Lex_Str);
               if Curr_Line.Context = In_Annotation then
                  Curr_Line.Anno_Context := Mid_Annotation;
               end if;

            elsif Next_Token = SP_Symbols.RWand then
               Check_Following_Token
                 (Prog_Text         => Prog_Text,
                  Possible_Follower => SP_Symbols.RWthen,
                  Transformed_Token => SP_Symbols.RWandthen,
                  Curr_Line         => Curr_Line,
                  Next_Token        => Next_Token,
                  Lex_Str           => Lex_Str);
            elsif Next_Token = SP_Symbols.RWor then
               Check_Following_Token
                 (Prog_Text         => Prog_Text,
                  Possible_Follower => SP_Symbols.RWelse,
                  Transformed_Token => SP_Symbols.RWorelse,
                  Curr_Line         => Curr_Line,
                  Next_Token        => Next_Token,
                  Lex_Str           => Lex_Str);
            elsif Next_Token = SP_Symbols.RWnot then
               Check_Following_Token
                 (Prog_Text         => Prog_Text,
                  Possible_Follower => SP_Symbols.RWin,
                  Transformed_Token => SP_Symbols.RWnotin,
                  Curr_Line         => Curr_Line,
                  Next_Token        => Next_Token,
                  Lex_Str           => Lex_Str);
            elsif Next_Token = SP_Symbols.RWfor then
               Check_Following_Token
                 (Prog_Text         => Prog_Text,
                  Possible_Follower => SP_Symbols.RWall,
                  Transformed_Token => SP_Symbols.RWforall,
                  Curr_Line         => Curr_Line,
                  Next_Token        => Next_Token,
                  Lex_Str           => Lex_Str);
               if Next_Token = SP_Symbols.RWfor then
                  Check_Following_Token
                    (Prog_Text         => Prog_Text,
                     Possible_Follower => SP_Symbols.RWsome,
                     Transformed_Token => SP_Symbols.RWforsome,
                     Curr_Line         => Curr_Line,
                     Next_Token        => Next_Token,
                     Lex_Str           => Lex_Str);
               end if;
            else
               Lex_Str := LexTokenManager.Null_String;
            end if;

         when SP_Symbols.string_literal =>
            Insert_String_Literal (Line_In   => Curr_Line.Conts,
                                   Start_Pos => Start_Posn,
                                   End_Pos   => End_Posn,
                                   Lex_Str   => Lex_Str);
         when SP_Symbols.character_literal | SP_Symbols.integer_number | SP_Symbols.real_number | SP_Symbols.based_integer =>
            Insert_Lex_String (The_Line  => Curr_Line.Conts,
                               Start_Pos => Start_Posn,
                               End_Pos   => End_Posn,
                               Lex_Str   => Lex_Str);
         when SP_Symbols.annotation_start | SP_Symbols.proof_context =>
            if Curr_Line.Context = In_Annotation then
               Next_Token := SP_Symbols.annotation_continuation;
            else
               LineManager.Set_Context (Curr_Line   => Curr_Line,
                                        New_Context => In_Annotation);
            end if;

            if Curr_Line.Anno_Context = Ended_Annotation then
               Curr_Line.Anno_Context := Start_Annotation;
            end if;

            Lex_Str := LexTokenManager.Null_String;

         when SP_Symbols.annotation_end =>
            LineManager.Set_Context (Curr_Line   => Curr_Line,
                                     New_Context => In_Ada);
            Lex_Str := LexTokenManager.Null_String;

         when SP_Symbols.hide_directive =>
            -- skip over hide
            LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                       Curr_Line => Curr_Line);

            --# accept F, 10, Next_Token, "Skipping so ineffective assignment";
            NextLex (Curr_Line => Curr_Line,
                     Token     => Next_Token);
            --# end accept;

            -- now get designator
            LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                       Curr_Line => Curr_Line);
            Start_Line := Curr_Line.Line_No;
            Start_Posn := Curr_Line.Curr_Pos;
            NextLex (Curr_Line => Curr_Line,
                     Token     => Next_Token);
            End_Posn := Curr_Line.Curr_Pos - 1;
            if Next_Token = SP_Symbols.identifier then
               Check_Reserved
                 (Curr_Line  => Curr_Line,
                  Start_Pos  => Start_Posn,
                  End_Pos    => End_Posn,
                  Look_Ahead => False,
                  Token      => Next_Token);
            end if;

            if Next_Token /= SP_Symbols.identifier then
               -- ???? Report Error ???? --
               ErrorHandler.Lex_Error
                 (Error_Message    => "Hide designator missing",
                  Recovery_Message => "Hide directive ignored",
                  Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                              Start_Pos     => Start_Posn),
                                                                 Token_Str => LexTokenManager.Null_String));
               LineManager.Reset_Curr_Pos (Curr_Line => Curr_Line);
               Next_Token := SP_Symbols.comment;
            else
               Hidden_Part     := True;
               Hide_Designator := LexTokenLists.Null_List;
               Insert_Lex_String (The_Line  => Curr_Line.Conts,
                                  Start_Pos => Start_Posn,
                                  End_Pos   => End_Posn,
                                  Lex_Str   => Lex_Str);
               LexTokenLists.Append (Hide_Designator, Lex_Str);

               loop
                  --# assert E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
                  --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
                  --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
                  --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
                  LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                             Curr_Line => Curr_Line);
                  exit when E_Strings.Get_Element (E_Str => Curr_Line.Conts,
                                                   Pos   => Curr_Line.Curr_Pos) /= '.';
                  LineManager.Lookahead_Char (Curr_Line => Curr_Line,
                                              Ch        => Ch);
                  LineManager.Reject_Lookahead (Curr_Line => Curr_Line);
                  exit when Ch = '.';   -- stop if ..
                  LineManager.Accept_Char (Curr_Line => Curr_Line); -- absorb dot
                  LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                             Curr_Line => Curr_Line);
                  Start_Line := Curr_Line.Line_No;
                  Start_Posn := Curr_Line.Curr_Pos;
                  NextLex (Curr_Line => Curr_Line,
                           Token     => Next_Token);
                  End_Posn := Curr_Line.Curr_Pos - 1;
                  if Next_Token = SP_Symbols.identifier then
                     Check_Reserved
                       (Curr_Line  => Curr_Line,
                        Start_Pos  => Start_Posn,
                        End_Pos    => End_Posn,
                        Look_Ahead => False,
                        Token      => Next_Token);
                  end if;
                  if Next_Token /= SP_Symbols.identifier then
                     -- ???? Report Error ???? --
                     ErrorHandler.Lex_Error
                       (Error_Message    => "Hide designator incomplete",
                        Recovery_Message => "Hide directive ignored",
                        Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                                    Start_Pos     => Start_Posn),
                                                                       Token_Str => LexTokenManager.Null_String));
                     LineManager.Reset_Curr_Pos (Curr_Line => Curr_Line);
                     Next_Token  := SP_Symbols.comment;
                     Hidden_Part := False;
                     exit;
                  end if;
                  Insert_Lex_String (The_Line  => Curr_Line.Conts,
                                     Start_Pos => Start_Posn,
                                     End_Pos   => End_Posn,
                                     Lex_Str   => Lex_Str);
                  LexTokenLists.Append (Hide_Designator, Lex_Str);
               end loop;
            end if;
            Lex_Str := LexTokenManager.Null_String;

         when SP_Symbols.apostrophe =>
            -- Check for attribute designator.
            LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                       Curr_Line => Curr_Line);
            if Ada.Characters.Handling.Is_Letter (E_Strings.Get_Element (E_Str => Curr_Line.Conts,
                                                                         Pos   => Curr_Line.Curr_Pos)) then
               Start_Line := Curr_Line.Line_No;
               Start_Posn := Curr_Line.Curr_Pos;
               GetIdent (Curr_Line => Curr_Line,
                         Token     => Next_Token);
               End_Posn := Curr_Line.Curr_Pos - 1;
               Insert_Lex_String (The_Line  => Curr_Line.Conts,
                                  Start_Pos => Start_Posn,
                                  End_Pos   => End_Posn,
                                  Lex_Str   => Lex_Str);
               if Next_Token = SP_Symbols.identifier then
                  Next_Token := SP_Symbols.attribute_ident;
               else
                  -- ???? Report Error ???? --
                  ErrorHandler.Lex_Error
                    (Error_Message    => "Attribute identifier expected",
                     Recovery_Message => "Attribute identifier assumed",
                     Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                                 Start_Pos     => Start_Posn),
                                                                    Token_Str => Lex_Str));
                  Next_Token := SP_Symbols.attribute_ident; -- Error recovery.
               end if;
            else
               Lex_Str := LexTokenManager.Null_String;
            end if;

         when SP_Symbols.illegal_id =>
            Next_Token := SP_Symbols.identifier;   -- Error recovery.
            Insert_Lex_String (The_Line  => Curr_Line.Conts,
                               Start_Pos => Start_Posn,
                               End_Pos   => End_Posn,
                               Lex_Str   => Lex_Str);
            -- ???? Report Error ???? --
            ErrorHandler.Lex_Error
              (Error_Message    => "Illegal identifier",
               Recovery_Message => "Identifier assumed",
               Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                           Start_Pos     => Start_Posn),
                                                              Token_Str => Lex_Str));

         when SP_Symbols.illegal_number =>
            Next_Token := SP_Symbols.integer_number;
            Insert_Lex_String (The_Line  => Curr_Line.Conts,
                               Start_Pos => Start_Posn,
                               End_Pos   => End_Posn,
                               Lex_Str   => Lex_Str);
            -- ???? Report Error ???? --
            if E_Strings.Get_Element (E_Str => Curr_Line.Conts,
                                      Pos   => Start_Posn) = '0' then
               ErrorHandler.Lex_Error
                 (Error_Message    => "Illegal number - possible mis-use of '0' instead of 'O' as first letter of identifier",
                  Recovery_Message => "Number assumed",
                  Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                              Start_Pos     => Start_Posn),
                                                                 Token_Str => Lex_Str));
            else
               ErrorHandler.Lex_Error
                 (Error_Message    => "Illegal number",
                  Recovery_Message => "Number assumed",
                  Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                              Start_Pos     => Start_Posn),
                                                                 Token_Str => Lex_Str));
            end if;

         when SP_Symbols.based_real =>
            Insert_Lex_String (The_Line  => Curr_Line.Conts,
                               Start_Pos => Start_Posn,
                               End_Pos   => End_Posn,
                               Lex_Str   => Lex_Str);
            -- ???? Report Error ???? --
            ErrorHandler.Lex_Error
              (Error_Message    => "Based real numbers are not allowed",
               Recovery_Message => "Number assumed",
               Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                           Start_Pos     => Start_Posn),
                                                              Token_Str => Lex_Str));

         when SP_Symbols.unterminated_string =>
            Next_Token := SP_Symbols.string_literal;  -- Error recovery.
            Insert_Lex_String (The_Line  => Curr_Line.Conts,
                               Start_Pos => Start_Posn,
                               End_Pos   => End_Posn,
                               Lex_Str   => Lex_Str);
            -- ???? Report Error ???? --
            ErrorHandler.Lex_Error
              (Error_Message    => "Unterminated string",
               Recovery_Message => "String assumed",
               Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                           Start_Pos     => Start_Posn),
                                                              Token_Str => Lex_Str));

         when SP_Symbols.illegal_comment =>
            Lex_Str    := LexTokenManager.Null_String;
            Next_Token := SP_Symbols.comment;  -- Error recovery.

            -- ???? Report Error ???? --
            ErrorHandler.Lex_Error
              (Error_Message    => "Illegal character in comment",
               Recovery_Message => "Illegal character ignored",
               Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                           Start_Pos     => Start_Posn),
                                                              Token_Str => LexTokenManager.Null_String));

         when SP_Symbols.illegal_token =>
            Lex_Str := LexTokenManager.Null_String;
            -- ???? Report Error ???? --
            ErrorHandler.Lex_Error
              (Error_Message    => "Illegal token",
               Recovery_Message => "Token ignored",
               Error_Item       => LexTokenManager.Lex_Value'(Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                                                                           Start_Pos     => Start_Posn),
                                                              Token_Str => LexTokenManager.Null_String));

         when SP_Symbols.semicolon =>
            Lex_Str := LexTokenManager.Null_String;
            if Curr_Line.Context = In_Annotation then
               Curr_Line.Anno_Context := Ended_Annotation;
            end if;

         when others =>
            Lex_Str := LexTokenManager.Null_String;
      end case;

      exit when Hidden_Part
        or else (Next_Token /= SP_Symbols.comment
                   and then Next_Token /= SP_Symbols.annotation_continuation
                   and then Next_Token /= SP_Symbols.illegal_token);
   end loop;

   --# assert E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;

   if Hidden_Part then
      End_Hide_Found := False;

      loop
         --# assert E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
         --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
         --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
         --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
         LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                    Curr_Line => Curr_Line);
         Start_Line := Curr_Line.Line_No;
         Start_Posn := Curr_Line.Curr_Pos;
         NextLex (Curr_Line => Curr_Line,
                  Token     => Next_Token);
         End_Posn := Curr_Line.Curr_Pos - 1;

         if Next_Token = SP_Symbols.SPEND then
            End_Hide_Found := True;
         elsif Next_Token = SP_Symbols.identifier then
            Check_Reserved
              (Curr_Line  => Curr_Line,
               Start_Pos  => Start_Posn,
               End_Pos    => End_Posn,
               Look_Ahead => False,
               Token      => Next_Token);
         end if;
         if Next_Token = SP_Symbols.RWend then
            LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                       Curr_Line => Curr_Line);
            Start_Line := Curr_Line.Line_No;
            Start_Posn := Curr_Line.Curr_Pos;
            NextLex (Curr_Line => Curr_Line,
                     Token     => Next_Token);
            End_Posn := Curr_Line.Curr_Pos - 1;

            if Next_Token = SP_Symbols.identifier then
               End_Designator := LexTokenLists.Null_List;
               Insert_Lex_String (The_Line  => Curr_Line.Conts,
                                  Start_Pos => Start_Posn,
                                  End_Pos   => End_Posn,
                                  Lex_Str   => Lex_Str);
               LexTokenLists.Append (End_Designator, Lex_Str);

               -- Process remainder of dotted name (if any)
               loop
                  --# assert E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
                  --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
                  --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
                  --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;
                  LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                             Curr_Line => Curr_Line);
                  exit when E_Strings.Get_Element (E_Str => Curr_Line.Conts,
                                                   Pos   => Curr_Line.Curr_Pos) /= '.';
                  LineManager.Lookahead_Char (Curr_Line => Curr_Line,
                                              Ch        => Ch);
                  LineManager.Reject_Lookahead (Curr_Line => Curr_Line);
                  exit when Ch = '.';   -- stop if '..'
                  LineManager.Accept_Char (Curr_Line => Curr_Line); -- absorb dot
                  LineManager.Next_Sig_Char (Prog_Text => Prog_Text,
                                             Curr_Line => Curr_Line);
                  Start_Posn := Curr_Line.Curr_Pos;
                  NextLex (Curr_Line => Curr_Line,
                           Token     => Next_Token); -- get expected identifier
                  End_Posn := Curr_Line.Curr_Pos - 1;
                  exit when Next_Token /= SP_Symbols.identifier;
                  Insert_Lex_String
                    (The_Line  => Curr_Line.Conts,
                     Start_Pos => Start_Posn,
                     End_Pos   => End_Posn,
                     Lex_Str   => Lex_Str);
                  LexTokenLists.Append (End_Designator, Lex_Str);
               end loop;

               if Next_Token = SP_Symbols.identifier then
                  End_Hide_Found := LexTokenLists.Eq_Unit (First_Item => Hide_Designator,
                                                           Second     => End_Designator);
               end if;
            end if;
         end if;

         if End_Hide_Found then
            Hidden_Part := False;
            Next_Token  := SP_Symbols.hide_directive;
            Lex_Str     := LexTokenManager.Null_String;
         end if;
         exit when not Hidden_Part;
      end loop;
   end if;

   --# assert E_Strings.Get_Length (Curr_Line.Conts) < Natural'Last and
   --#   Curr_Line.Curr_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Lookahead_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1 and
   --#   Curr_Line.Last_Token_Pos <= E_Strings.Get_Length (Curr_Line.Conts) + 1;

   Token       := Next_Token;
   Lex_Val     :=
     LexTokenManager.Lex_Value'
     (Position  => LexTokenManager.Token_Position'(Start_Line_No => Start_Line,
                                                   Start_Pos     => Start_Posn),
      Token_Str => Lex_Str);
   Punct_Token := Is_Punct_Token (Token => Next_Token);
end Lex;
