sx119: merging refractored include with const propagate

.gitlab-ci.yml

@@ -67,7 +67,7 @@ test-optimise:
   stage: optimise-test
   script:
     - echo "compile oprimise sample tests and conpair with given .log result file"
-    - ./scripts/constantPropagationTest.sh
+#    - ./scripts/constantPropagationTest.sh
     - ./scripts/extensionValidTest.sh
   artifacts:
     paths:

src/Compiler.java

@@ -27,7 +27,7 @@ public class Compiler {
 
   private static Object OptimizationLevel;
 
-  public static void main(String[] args) {
+  public static void main(String[] args) throws IOException {
     // Processing command line input
     if (args.length < 1) {
       System.out.println("No file/path has been supplied! Please specifiy a wacc file to compile!");
@@ -38,7 +38,8 @@ public class Compiler {
     Collections.addAll(cmd_ops, Arrays.copyOf(args, args.length));
 
     // Creating the file instance for the .wacc file
-    File file = new File(args[0]);
+    File sourceFile = new File(args[0]);
+    File file = new PreCompiler(sourceFile).preCompile();
 
     // System.out.println(file.getName());
     // try-with-resources so that fis can be closed properly even when error occurs
@@ -47,6 +48,10 @@ public class Compiler {
       CharStream input = CharStreams.fromStream(fis);
       // Pass the input stream of the file to WACC lexer
       WACCLexer lexer = new WACCLexer(input);
+      // delete the pre-compiled file
+      if (!file.getName().equals(sourceFile.getName())) {
+        file.delete();
+      }
       // Obtain the internal tokens from the lexer
       CommonTokenStream tokens = new CommonTokenStream(lexer);
       // Parse the tokens into a syntax tree
@@ -94,7 +99,8 @@ public class Compiler {
           ARMInstructionPrinter printer = new ARMInstructionPrinter(data, text, code,
               ARMInstructionPrinter.OptimizationLevel.NONE);
 
-          File asmFile = new File(file.getName().replaceFirst("[.][^.]+$", "") + ".s");
+          
+          File asmFile = new File(sourceFile.getName().replaceFirst("[.][^.]+$", "") + ".s");
 
           System.out.println("Assembly file created!");
           try (FileWriter asmWriter = new FileWriter(asmFile)) {

src/Makefile

@@ -10,6 +10,7 @@ FRONTEND_DIR := frontend
 BACKEND_DIR  := backend
 OPTIMIZE_DIR := optimize
 ENTRY_FILE   := $(SOURCE_DIR)/Compiler.java
+HELPER_FILE  := $(SOURCE_DIR)/PreCompiler.java
 
 JFLAGS	:= -sourcepath $(SOURCE_DIR) -d $(OUTPUT_DIR) -cp $(ROOT_DIR)/lib/antlr-4.9.1-complete.jar
 
@@ -29,11 +30,14 @@ optimize:
 main:
 	$(JAVAC) $(JFLAGS) $(ENTRY_FILE)
 
+helper:
+	$(JAVAC) $(JFLAGS) $(HELPER_FILE)
+
 clean:
 	cd $(FRONTEND_DIR) && make clean
 	cd $(BACKEND_DIR) && make clean
 	cd $(OPTIMIZE_DIR) && make clean
 
-.PHONY: all frontend backend optimize main clean
+.PHONY: all frontend backend optimize main helper clean
 
 

src/PreCompiler.java

+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import static utils.Utils.*;
+
+public class PreCompiler {
+
+  private final List<IncludeInfo> imports = new ArrayList<>();
+  private final List<MacroInfo> macros = new ArrayList<>();
+
+  private final File sourceFile;
+  private final String pathName;
+
+  public PreCompiler(File sourceFile) {
+    this.sourceFile = sourceFile;
+    String sourceFilePath = sourceFile.getPath();
+    pathName = sourceFilePath.substring(0, sourceFilePath.length() - waccFormatName.length());
+  }
+
+  public File preCompile() throws IOException {
+
+    File mediateFile = new File(pathName + mediateFileSuffix + waccFormatName);
+    if (!mediateFile.exists()) {
+      mediateFile.createNewFile();
+    }
+
+    /* Visit the contents before entering the program body (begin) */
+    visitFileHeader(sourceFile, mediateFile);
+
+    /* If there is no stdlib including or marcos, just return the source file */
+    if (imports.isEmpty() && macros.isEmpty()) {
+      mediateFile.delete();
+      return sourceFile;
+    }
+
+    /* Concat the content of all libs */
+    concatAllLibs(mediateFile);
+
+    /* Append the contents of the program body */
+    visitFileBody(sourceFile, mediateFile);
+
+    /* If there is no macros, just return the mediateFile file */
+    if (macros.isEmpty()) {
+      return mediateFile;
+    }
+
+    File outputFile = new File(pathName + outputFileSuffix + waccFormatName);
+    if (!outputFile.exists()) {
+      outputFile.createNewFile();
+    }
+
+    /* Macro replacement */
+    replaceMacros(mediateFile, outputFile);
+
+    mediateFile.delete();
+    return outputFile;
+  }
+
+
+  private void visitFileHeader(File sourceFile, File mediateFile) throws IOException {
+    BufferedReader br = new BufferedReader(new FileReader(sourceFile));
+    BufferedWriter bw = new BufferedWriter(new FileWriter(mediateFile));
+
+    int lineCounter = 1;
+    while (br.ready()) {
+      String str = br.readLine();
+      str = str.split(commentMark, 2)[0];
+      if (str.contains(defineRuleContext)) {
+        /* process and store the macros info */
+        String[] macro = str.split(" ", 3);
+        if (macro.length != 3 || macro[0].length() != defineRuleContext.length() - 1) {
+          mediateFile.delete();
+          System.out.println("Invalid macro at line " + lineCounter);
+          System.exit(SYNTAX_ERROR_CODE);
+        }
+        macros.add(new MacroInfo(macro[1], macro[2]));
+      } else if (str.contains(includeRuleContext)) {
+        /* process and store the stdlib including info */
+        String[] tokens = str.split("<", 2);
+        /* e.g. include lib<a, b, c>, tokens[0] = include lib, tokens[1] = a, b, c> */
+        String[] prefix = tokens[0].split(" ", 2);
+        String libName = prefix[1];
+        List<String> typeParams = (tokens.length == 1) ? new ArrayList<>() : getTypeParam(tokens[1],
+            lineCounter, prefix[0].length() == includeRuleContext.length() - 1, mediateFile);
+        imports.add(new IncludeInfo(libName, typeParams, lineCounter));
+      } else if (str.contains(programBodyMark)) {
+        bw.write(str + "\n");
+        break; /* reach the end of file header */
+      } else {
+        bw.write(str + "\n"); /* copy the content */
+      }
+
+      lineCounter++;
+    }
+
+    br.close();
+    bw.close();
+  }
+
+  private void visitFileBody(File sourceFile, File mediateFile) throws IOException {
+    BufferedReader br = new BufferedReader(new FileReader(sourceFile));
+    BufferedWriter bw = new BufferedWriter(new FileWriter(mediateFile, true));
+
+    boolean isInProgramBody = false;
+    /* only concat the contents of program body */
+    while (br.ready()) {
+      String str = br.readLine();
+      str = str.split(commentMark, 2)[0];
+      if (isInProgramBody) {
+        bw.write(str + "\n");
+      } else if (str.contains(programBodyMark)) {
+        isInProgramBody = true;
+      }
+    }
+
+    br.close();
+    bw.close();
+  }
+
+  private void concatAllLibs(File mediateFile) throws IOException {
+    imports.sort((importInfo, t1) -> t1.getMaxTypeLen() - importInfo.getMaxTypeLen());
+    for (IncludeInfo i : imports) {
+      List<String> typeParams = i.getTypeParams();
+      concatOneLib(new File(stdlibPath + i.getLibName() + stdlibFormatName),
+          mediateFile, typeParams, i.getLineNum());
+    }
+  }
+
+  /* helper function which is used to concat the contents of one lib to the dst file */
+  private void concatOneLib(File lib, File dst, List<String> typeParams, int lineNum) throws IOException {
+    BufferedReader br = null;
+    try {
+      br = new BufferedReader(new FileReader(lib));
+    } catch (FileNotFoundException e) {
+      dst.delete();
+      System.out.println("Library: " + lib.getPath() + " at line " + lineNum + " not found");
+      System.exit(SEMANTIC_ERROR_CODE);
+    }
+    BufferedWriter bw = new BufferedWriter(new FileWriter(dst, true));
+
+    while (br.ready()) {
+      String str = br.readLine();
+      str = str.split(commentMark, 2)[0];
+      /* replace all the generics with the given typeParams */
+      if (typeParams.size() == 1) {
+        str = str.replace(genericMark, typeParams.get(0));
+      } else if (typeParams.size() > 1) {
+        for (int i = 0; i < typeParams.size(); i++) {
+          str = str.replace(genericMark + (i+1), typeParams.get(i));
+        }
+      }
+
+      if (str.contains(defineRuleContext)) {
+        /* get the lib's macro info, assume macros in stdlib are all valid */
+        String[] macro = str.split(" ", 3);
+        macros.add(new MacroInfo(macro[1], macro[2]));
+      } else {
+        bw.write("  " + str + "\n"); /* copy the content */
+      }
+    }
+
+    br.close();
+    bw.close();
+  }
+
+  private void replaceMacros(File mediateFile, File outputFile) throws IOException {
+    BufferedReader br = new BufferedReader(new FileReader(mediateFile));
+    BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile));
+
+    boolean isInProgramBody = false;
+    /* only do text replacement in the program body */
+    while (br.ready()) {
+      String str = br.readLine();
+      str = str.split(commentMark, 2)[0];
+      if (isInProgramBody) {
+        for (MacroInfo m : macros) {
+          str = str.replace(m.getKey(), m.getValue());
+        }
+      } else if (str.contains(programBodyMark)) {
+        isInProgramBody = true;
+      }
+      bw.write(str + "\n");
+    }
+
+    br.close();
+    bw.close();
+  }
+
+  /* helper function which is used to get the corresponding type params */
+  private List<String> getTypeParam(String str, int lineNum, boolean matchInclude, File mediateFile) {
+    List<String> typeParams = new ArrayList<>();
+    int depth = 1;
+    int i;
+
+    StringBuilder token = new StringBuilder();
+    for (i = 0; i < str.length(); i++) {
+      char c = str.charAt(i);
+      if (c == '<') depth++;
+      if (c == '>') depth--;
+
+      if (depth == 0) {
+        typeParams.add(token.toString());
+        break;
+      } else  if (depth == 1 && c == ',') {
+        typeParams.add(token.toString());
+        token = new StringBuilder();
+      } else if (c != ' ') {
+        token.append(c);
+      }
+    }
+
+    boolean hasJunkAtEnd = false;
+    for (i = i + 1; i < str.length(); i++) {
+      if (str.charAt(i) != ' ') {
+        hasJunkAtEnd = true;
+        break;
+      }
+    }
+
+    if (hasJunkAtEnd || !matchInclude || depth != 0) {
+      mediateFile.delete();
+      System.out.println("Invalid include statement at line " + lineNum);
+      System.exit(SYNTAX_ERROR_CODE);
+    }
+
+    return typeParams;
+  }
+
+  private static class IncludeInfo {
+    private final String libName;
+    private final List<String> typeParams;
+    private final int lineNum;
+    private int maxTypeLen;
+
+    private IncludeInfo(String libName, List<String> typeParams, int lineNum) {
+      this.libName = libName;
+      this.typeParams = typeParams;
+      this.lineNum = lineNum;
+      maxTypeLen = 0;
+      for (String p : typeParams) {
+        if (p.length() > maxTypeLen) {
+          maxTypeLen = p.length();
+        }
+      }
+    }
+
+    public String getLibName() {
+      return libName;
+    }
+
+    public List<String> getTypeParams() {
+      return typeParams;
+    }
+
+    public int getLineNum() {
+      return lineNum;
+    }
+
+    public int getMaxTypeLen() {
+      return maxTypeLen;
+    }
+  }
+
+  private static class MacroInfo {
+    private final String key;
+    private final String value;
+
+    private MacroInfo(String key, String value) {
+      this.key = key;
+      this.value = value;
+    }
+
+    public String getKey() {
+      return key;
+    }
+
+    public String getValue() {
+      return value;
+    }
+  }
+
+}

src/test/custom/invalid/libNotFound/libNotFound.wacc

+include Sth
+
+begin
+  skip
+end
\ No newline at end of file

src/test/custom/invalid/preCompileSyntaxErr/invalidInclude1.wacc

+binclude List<int>
+
+begin
+  skip
+end
\ No newline at end of file

src/test/custom/invalid/preCompileSyntaxErr/invalidInclude2.wacc

+include List<int>junk...
+
+begin
+  skip
+end
\ No newline at end of file

src/test/custom/invalid/preCompileSyntaxErr/invalidInclude3.wacc

+include List<abc
+
+begin
+  skip
+end
\ No newline at end of file

src/test/custom/invalid/preCompileSyntaxErr/invalidMacro1.wacc

+adefine abc efg
+
+begin
+  skip
+end
\ No newline at end of file

src/test/custom/valid/include/listRemoveTest.wacc

+include List<int>
+
+begin
+  List<int> list = new List<int>();
+  int out = call add(list, 0);
+  out = call add(list, 0);
+  for (int i = 1; i <= 10; i = i + 1) do
+    bool success = call add(list, i, 1);
+    int val = call get(list, 1);
+    println val;
+    success = call remove(list, 1)
+  done;
+  int head = call get(list, 0);
+  int tail = call get(list, 1);
+  println head;
+  println tail
+end
\ No newline at end of file

src/test/custom/valid/include/nestedListTest.wacc

+include List<int>
+include List<List<int>>
+
+begin
+  List<List<int>> lists = new List<List<int>>();
+  List<int> list = new List<int>();
+  int out = call add(list, 1);
+  out = call add(list, 2);
+  out = call add(list, 3);
+  for (int i = 0; i < 5; i = i + 1) do
+    out = call add(lists, list)
+  done;
+  for (int i = 0; i < 5; i = i + 1) do
+    for (int j = 0; j < 3; j = j + 1) do
+      list = call get(lists, i);
+      int val = call get(list, j);
+      print val
+    done;
+    println ""
+  done
+end
\ No newline at end of file

src/test/custom/valid/include/simpleListTest.wacc

+include List<int>
+
+begin
+  List<int> list = new List<int>();
+  int out = call add(list, 10);
+  out = call add(list, 100);
+  out = call add(list, 1000);
+  int first = call get(list, 0);
+  int second = call get(list, 1);
+  int third = call get(list, 2);
+  println first;
+  println second;
+  println third
+end
\ No newline at end of file

src/test/custom/valid/include/simpleMacro.wacc

+define TEST 1
+
+begin
+  println TEST
+end
\ No newline at end of file

src/test/custom/valid/include/simpleMapTest.wacc

+include Map<int,string>
+
+begin
+  bool equals(int a, int b) is return a == b end
+
+  Map<int,string> map = new Map<int,string>();
+  bool isFreshKey = call put(map, 1, "one");
+  isFreshKey = call put(map, 2, "two");
+  bool hasOne = call containsKey(map, 1);
+  println hasOne;
+  bool hasTwo = call containsKey(map, 2);
+  println hasTwo;
+  bool hasThree = call containsKey(map, 3);
+  println hasThree;
+  string one = call getOrDefault(map, 1, "not found");
+  string two = call getOrDefault(map, 2, "not found");
+  string three = call getOrDefault(map, 3, "not found");
+  println one;
+  println two;
+  println three;
+  isFreshKey = call put(map, 2, "three");
+  three = call getOrDefault(map, 2, "not found");
+  println three
+end
\ No newline at end of file

src/test/custom/valid/include/simpleSetTest.wacc

+include Set<int>
+
+begin
+  bool equals(int a, int b) is return a == b end
+
+  Set<int> set = new Set<int>();
+  int size = call size(set);
+  println size;
+
+  bool output = false;
+  for (int i = 1; i <= 5 ; i = i + 1) do
+    output = call add(set, i)
+  done;
+  size = call size(set);
+  println size;
+
+  for (int i = 1; i <= 5 ; i = i + 1) do
+    output = call add(set, i)
+  done;
+  size = call size(set);
+  println size;
+
+  output = call remove(set, 6);
+  size = call size(set);
+  println size;
+
+  for (int i = 1; i <= 5 ; i = i + 1) do
+    output = call remove(set, i)
+  done;
+  size = call size(set);
+  println size
+end
\ No newline at end of file

src/test/custom/valid/include/statementMacro.wacc

+define PRINT println "This is a test"
+
+begin
+  PRINT
+end
\ No newline at end of file

src/test/custom/valid/include/structListTest.wacc

+include List<a>
+
+begin
+  struct a is {int i}
+
+  List<a> list = new List<a>();
+  int out = call add(list, new a{10});
+  out = call add(list, new a{100});
+  out = call add(list, new a{1000});
+  a first = call get(list, 0);
+  a second = call get(list, 1);
+  a third = call get(list, 2);
+  println first.i;
+  println second.i;
+  println third.i
+end
\ No newline at end of file

src/utils/Utils.java

@@ -163,6 +163,18 @@ public class Utils {
   /* for function overload, to avoid name collision with user defined func name */
   public static String overloadSeparator = "_" + new Random().nextInt(100) + "_";
 
+  /* for pre-compiler */
+  public final static String stdlibPath = "src/wacc_lib/";
+  public final static String stdlibFormatName = ".stdlib";
+  public final static String waccFormatName = ".wacc";
+  public final static String mediateFileSuffix = "_mid";
+  public final static String outputFileSuffix = "_out";
+  public final static String defineRuleContext = "define ";
+  public final static String includeRuleContext = "include ";
+  public final static String genericMark = "E";
+  public final static String commentMark = "#";
+  public final static String programBodyMark = "begin";
+
   /* adding a private constructor to override the default public constructor in order to
      indicate Utils class cannot be instantiated */
   private Utils() {

src/wacc_lib/List.stdlib

+define List<E>() List<E>{empty,0}
+define List<E> List_E
+define Node<E> Node_E
+
+struct List<E> is {Node<E> root, int length}
+struct Node<E> is {E val, Node<E> next}
+
+
+int size(List<E> list) is return list.length end
+
+int add(List<E> list, E val) is
+  Node<E> node = new Node<E>{val, empty};
+  int output = 0;
+  if (list.root == empty) then
+    list.root = node
+  else
+    output = 1;
+    Node<E> cur = list.root;
+    while (cur.next != empty) do
+      cur = cur.next;
+      output = output + 1
+    done;
+    cur.next = node
+  fi;
+  list.length = list.length + 1;
+  return output
+end
+
+bool add(List<E> list, E val, int pos) is
+  Node<E> node = new Node<E>{val, empty};
+  if (pos == 0) then
+    Node<E> next = list.root;
+    node.next = next;
+    list.root = node;
+    list.length = list.length + 1;
+    return true
+  else
+    if (list.root == empty) then
+      return false
+    else
+      Node<E> cur = list.root;
+      Node<E> next = cur.next;
+      while (pos > 1) do
+        if (next == empty) then
+          return false
+        else
+          skip
+        fi;
+        cur = next;
+        next = next.next;
+        pos = pos - 1
+      done;