New java source parser for declaration locations (experimental)

Summary:
The current source parser is based on ocamllex only.
In order to track field declaration locations, we propose a
new parser using ocamllex/menhir. This is a more ambitious
project that closely follows the official Java syntax.

Reviewed By: jvillard

Differential Revision: D24858280

fbshipit-source-id: 22d6766e5

Author: davidpichardie

infer/man/man1/infer-full.txt

@@ -1566,6 +1566,10 @@ INTERNAL OPTIONS
        --java-jar-compiler-reset
            Cancel the effect of --java-jar-compiler.
 
+       --java-source-parser-experimental
+           Activates: The experimental Java source parser for declaration
+           locations. (Conversely: --no-java-source-parser-experimental)
+
        --java-version-reset
            Cancel the effect of --java-version.
 

infer/src/base/Config.ml

@@ -1460,6 +1460,11 @@ and java_jar_compiler =
     ~meta:"path" "Specify the Java compiler jar used to generate the bytecode"
 
 
+and java_source_parser_experimental =
+  CLOpt.mk_bool ~long:"java-source-parser-experimental"
+    "The experimental Java source parser for declaration locations."
+
+
 and java_version =
   CLOpt.mk_int_opt ~long:"java-version" ?default:Version.java_version
     ~in_help:InferCommand.[(Capture, manual_java); (Analyze, manual_java)]
@@ -2927,6 +2932,8 @@ and java_debug_source_file_info = !java_debug_source_file_info
 
 and java_jar_compiler = !java_jar_compiler
 
+and java_source_parser_experimental = !java_source_parser_experimental
+
 and java_version = !java_version
 
 and javac_classes_out = !javac_classes_out

infer/src/base/Config.mli

@@ -344,6 +344,8 @@ val java_debug_source_file_info : string option
 
 val java_jar_compiler : string option
 
+val java_source_parser_experimental : bool
+
 val java_version : int option
 
 val javac_classes_out : string

infer/src/infer.ml

@@ -157,7 +157,9 @@ let () =
   | _ when Config.test_determinator && not Config.process_clang_ast ->
       TestDeterminator.compute_and_emit_test_to_run ()
   | _ when Option.is_some Config.java_debug_source_file_info ->
-      JSourceFileInfo.debug_on_file (Option.value_exn Config.java_debug_source_file_info)
+      if Config.java_source_parser_experimental then
+        JSourceLocations.debug_on_file (Option.value_exn Config.java_debug_source_file_info)
+      else JSourceFileInfo.debug_on_file (Option.value_exn Config.java_debug_source_file_info)
   | Analyze ->
       run Driver.Analyze
   | Capture | Compile | Run ->

infer/src/java/JavaFrontendStubs.ml

@@ -16,3 +16,7 @@ end
 module JSourceFileInfo = struct
   let debug_on_file _ = ()
 end
+
+module JSourceLocations = struct
+  let debug_on_file _ = ()
+end

infer/src/java/JavaFrontendStubs.mli

@@ -22,3 +22,7 @@ end
 module JSourceFileInfo : sig
   val debug_on_file : string -> unit
 end
+
+module JSourceLocations : sig
+  val debug_on_file : string -> unit
+end

infer/src/java/dune.in

@@ -7,7 +7,8 @@
  *)
 (* NOTE: prepend dune.common to this file! *)
 
-let lexer = "(ocamllex jSourceFileInfo)"
+let lexer = "(ocamllex jSourceLexer jSourceFileInfo)"
+let jparser = "(menhir (modules jSourceParser))"
 
 let java =
   Printf.sprintf
@@ -30,4 +31,4 @@ let java =
 let documentation = "(documentation (package infer) (mld_files JavaFrontend))"
 
 ;;
-Jbuild_plugin.V1.send (String.concat "\n" [lexer; java; documentation])
+Jbuild_plugin.V1.send (String.concat "\n" [lexer; jparser; java; documentation])

infer/src/java/jFrontend.ml

@@ -217,7 +217,9 @@ let compute_source_icfg program tenv source_basename package_opt source_file =
     if test node then try procedure cn node with Bir.Subroutine -> ()
   in
   (* we must set the java location for all classes in the source file before translation *)
-  JSourceFileInfo.collect_class_location program source_file ;
+  if Config.java_source_parser_experimental then
+    JSourceLocations.collect_class_location program source_file
+  else JSourceFileInfo.collect_class_location program source_file ;
   let () =
     JBasics.ClassMap.iter
       (select

infer/src/java/jSourceAST.ml

@@ -0,0 +1,43 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open! IStd
+
+type location = {line: int; col: int}
+
+type class_kind = Class of string | Interface of string | AnonymousClass | Enum of string
+
+type class_or_interface =
+  {location: location; kind: class_kind; inner_elements: class_or_interface list}
+
+type file_content = {package: string option; classes: class_or_interface list}
+
+type context = {prefix: string; mutable counter: int}
+
+let name_of_kind context = function
+  | Class id | Interface id | Enum id ->
+      id
+  | AnonymousClass ->
+      context.counter <- context.counter + 1 ;
+      string_of_int context.counter
+
+
+let rec iter ~action_on_class_location context {location; kind; inner_elements} =
+  let previous_prefix = context.prefix in
+  let name = name_of_kind context kind in
+  let context = {prefix= Printf.sprintf "%s%s$" context.prefix name; counter= 0} in
+  let classname = previous_prefix ^ name in
+  let col = location.col in
+  let line = location.line in
+  let _ = action_on_class_location ~classname ~col ~line in
+  List.iter inner_elements ~f:(iter ~action_on_class_location context)
+
+
+let iter_on_declarations ~action_on_class_location {package; classes} =
+  let prefix = Option.fold ~init:"" ~f:(fun _ s -> s ^ ".") package in
+  let context = {prefix; counter= 0} in
+  List.iter classes ~f:(iter ~action_on_class_location context)

infer/src/java/jSourceAST.mli

@@ -0,0 +1,20 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open! IStd
+
+type location = {line: int; col: int}
+
+type class_kind = Class of string | Interface of string | AnonymousClass | Enum of string
+
+type class_or_interface =
+  {location: location; kind: class_kind; inner_elements: class_or_interface list}
+
+type file_content = {package: string option; classes: class_or_interface list}
+
+val iter_on_declarations :
+  action_on_class_location:(classname:string -> col:int -> line:int -> unit) -> file_content -> unit