compiler-design-eth/test/cd/AbstractTestAgainstFrozenReference.java

281 lines
8.4 KiB
Java
Raw Normal View History

2020-01-15 22:34:57 +01:00
package cd;
2020-01-15 22:18:07 +01:00
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
2020-01-15 22:34:57 +01:00
import java.io.FileWriter;
2020-01-15 22:18:07 +01:00
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;
2020-01-15 22:34:57 +01:00
import java.util.concurrent.TimeUnit;
2020-01-15 22:18:07 +01:00
import org.junit.Assert;
2020-01-15 22:34:57 +01:00
import org.junit.Rule;
2020-01-15 22:18:07 +01:00
import org.junit.Test;
2020-01-15 22:34:57 +01:00
import org.junit.rules.DisableOnDebug;
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
2020-01-15 22:18:07 +01:00
2020-01-15 22:34:57 +01:00
import cd.backend.codegen.AssemblyFailedException;
2020-01-15 22:18:07 +01:00
import cd.frontend.parser.ParseFailure;
2020-01-15 22:32:25 +01:00
import cd.frontend.semantic.SemanticFailure;
2020-01-15 22:18:07 +01:00
import cd.ir.Ast.ClassDecl;
import cd.util.FileUtil;
import cd.util.debug.AstDump;
abstract public class AbstractTestAgainstFrozenReference {
public static final String SEMANTIC_OK = "OK";
public static final String PARSE_FAILURE = "ParseFailure";
public File file, sfile, binfile, infile;
2020-01-15 22:34:57 +01:00
public File parserreffile, semanticreffile, execreffile, cfgreffile, rdreffile,
nnreffile, optreffile;
2020-01-15 22:18:07 +01:00
public File errfile;
public Main main;
public static int counter = 0;
2020-01-15 22:34:57 +01:00
@Rule
public TestRule timeout = new DisableOnDebug(new Timeout(15, TimeUnit.SECONDS));
@Test
2020-01-15 22:18:07 +01:00
public void test() throws Throwable {
System.err.println("[" + counter++ + " = " + file + "]");
try {
// Delete intermediate files from previous runs:
if (sfile.exists())
sfile.delete();
if (binfile.exists())
binfile.delete();
runReference();
List<ClassDecl> astRoots = testParser();
if (astRoots != null) {
2020-01-15 22:32:25 +01:00
{
boolean passedSemanticAnalysis = testSemanticAnalyzer(astRoots);
2020-01-15 22:34:57 +01:00
{
if (passedSemanticAnalysis) {
testCodeGenerator(astRoots);
}
}
2020-01-15 22:32:25 +01:00
}
2020-01-15 22:18:07 +01:00
}
} catch (org.junit.ComparisonFailure cf) {
throw cf;
} catch (Throwable e) {
PrintStream err = new PrintStream(errfile);
err.println("Debug information for file: " + this.file);
err.println(this.main.debug.toString());
err.println("Test failed because an exception was thrown:");
err.println(" " + e.getLocalizedMessage());
err.println("Stack trace:");
e.printStackTrace(err);
System.err.println(FileUtil.read(errfile));
throw e;
}
// if we get here, then the test passed, so delete the errfile:
// (which has been accumulating debug output etc)
if (errfile.exists())
errfile.delete();
}
private void runReference() throws IOException, InterruptedException {
String slash = File.separator;
String colon = File.pathSeparator;
String javaExe = System.getProperty("java.home") + slash + "bin" + slash + Config.JAVA_EXE;
ProcessBuilder pb = new ProcessBuilder(
javaExe, "-Dcd.meta_hidden.Version=" + referenceVersion(),
2020-01-15 22:34:57 +01:00
"-cp", "lib/frozenReferenceObf.jar" + colon + " lib/junit-4.12.jar" + colon + "lib/antlr-4.7.1-complete.jar",
2020-01-15 22:18:07 +01:00
"cd.FrozenReferenceMain", file.getAbsolutePath());
Process proc = pb.start();
proc.waitFor();
try (InputStream err = proc.getErrorStream()) {
if (err.available() > 0) {
byte b[] = new byte[err.available()];
err.read(b, 0, b.length);
System.err.println(new String(b));
}
}
}
private static String referenceVersion() {
{
2020-01-15 22:34:57 +01:00
return "CD_HW_CODEGEN_FULL_SOL";
2020-01-15 22:18:07 +01:00
}
}
2020-01-15 22:34:57 +01:00
private String tryReadRefFile(File fileToFind) {
if (fileToFind.exists() && fileToFind.lastModified() >= file.lastModified()) {
try {
return FileUtil.read(fileToFind);
} catch (IOException e) {
throw new RuntimeException("ERROR: could not read file " + fileToFind.getPath());
}
}
throw new RuntimeException("ERROR: could not find file " + fileToFind.getPath());
}
2020-01-15 22:18:07 +01:00
/** Run the parser and compare the output against the reference results */
private List<ClassDecl> testParser() throws Exception {
2020-01-15 22:34:57 +01:00
String parserRef = tryReadRefFile(parserreffile);
2020-01-15 22:18:07 +01:00
List<ClassDecl> astRoots = null;
String parserOut;
try {
astRoots = main.parse(new FileReader(this.file));
parserOut = AstDump.toString(astRoots);
} catch (ParseFailure pf) {
{
2020-01-15 22:30:09 +01:00
// Parse errors are ok too.
main.debug("");
main.debug("");
main.debug("%s", pf.toString());
parserOut = PARSE_FAILURE;
2020-01-15 22:18:07 +01:00
}
}
{
2020-01-15 22:32:25 +01:00
// Now that the 2nd assignment is over, we don't
// do a detailed comparison of the AST, just check
// whether the parse succeeded or failed.
if (parserOut.equals(PARSE_FAILURE) || parserRef.equals(PARSE_FAILURE))
assertEquals("parser", parserRef, parserOut);
2020-01-15 22:18:07 +01:00
}
return astRoots;
}
2020-01-15 22:32:25 +01:00
private boolean testSemanticAnalyzer(List<ClassDecl> astRoots)
throws IOException {
String semanticRef = findSemanticRef();
boolean passed;
String result;
try {
main.semanticCheck(astRoots);
result = SEMANTIC_OK;
passed = true;
} catch (SemanticFailure sf) {
result = sf.cause.name();
main.debug("Error message: %s", sf.getLocalizedMessage());
passed = false;
}
assertEquals("semantic", semanticRef, result);
return passed;
}
private String findSemanticRef() throws IOException {
// Semantic ref file is a little different. It consists
// of 2 lines, but only the first line is significant.
// The second one contains additional information that we log
// to the debug file.
2020-01-15 22:34:57 +01:00
String res = tryReadRefFile(semanticreffile);
2020-01-15 22:32:25 +01:00
// Extract the first line: there should always be multiple lines,
// but someone may have tinkered with the file or something
if (res.contains("\n")) {
int newline = res.indexOf("\n");
String info = res.substring(newline + 1);
if (!info.equals("") && !info.equals("\n"))
main.debug("Error message from reference is: %s", info);
return res.substring(0, newline); // 1st line
} else {
return res;
}
}
2020-01-15 22:18:07 +01:00
2020-01-15 22:34:57 +01:00
/**
* Run the code generator, assemble the resulting .s file, and (if the output
* is well-defined) compare against the expected output.
*/
private void testCodeGenerator(List<ClassDecl> astRoots)
throws IOException {
// Determine the input and expected output.
String inFile = (infile.exists() ? FileUtil.read(infile) : "");
String execRef = tryReadRefFile(execreffile);
// Run the code generator:
try (FileWriter fw = new FileWriter(this.sfile)) {
main.generateCode(astRoots, fw);
}
// At this point, we have generated a .s file and we have to compile
// it to a binary file. We need to call out to GCC or something
// to do this.
String asmOutput = FileUtil.runCommand(
Config.ASM_DIR,
Config.ASM,
new String[] { binfile.getAbsolutePath(),
sfile.getAbsolutePath() }, null, false);
// To check if gcc succeeded, check if the binary file exists.
// We could use the return code instead, but this seems more
// portable to other compilers / make systems.
if (!binfile.exists())
throw new AssemblyFailedException(asmOutput);
// Execute the binary file, providing input if relevant, and
// capturing the output. Check the error code so see if the
// code signaled dynamic errors.
String execOut = FileUtil.runCommand(new File("."),
new String[] { binfile.getAbsolutePath() }, new String[] {},
inFile, true);
// Compute the output to what we expected to see.
if (!execRef.equals(execOut))
assertEqualOutput("exec", execRef, execOut);
}
2020-01-15 22:18:07 +01:00
private void assertEquals(String phase, String exp, String act_) {
String act = act_.replace("\r\n", "\n"); // for windows machines
if (!exp.equals(act)) {
warnAboutDiff(phase, exp, act);
}
}
2020-01-15 22:32:25 +01:00
2020-01-15 22:34:57 +01:00
/**
* Compare the output of two executions
*/
private void assertEqualOutput(String phase, String exp, String act_) {
String act = act_.replace("\r\n", "\n"); // for windows machines
if (!exp.equals(act)) {
warnAboutDiff(phase, exp, act);
}
}
2020-01-15 22:18:07 +01:00
private void warnAboutDiff(String phase, String exp, String act) {
try {
try (PrintStream err = new PrintStream(errfile)) {
err.println("Debug information for file: " + this.file);
err.println(this.main.debug.toString());
err.println(String.format(
"Phase %s failed because we expected to see:", phase));
err.println(exp);
err.println("But we actually saw:");
err.println(act);
err.println("The difference is:");
err.println(Diff.computeDiff(exp, act));
}
} catch (FileNotFoundException exc) {
System.err.println("Unable to write debug output to " + errfile
+ ":");
exc.printStackTrace();
}
Assert.assertEquals(
String.format("Phase %s for %s failed!", phase,
file.getPath()), exp, act);
}
}