diff --git a/src/main/java/com/huawei/unt/UNTMain.java b/src/main/java/com/huawei/unt/UNTMain.java index d5ede36244a3d49aada3c7124d1f83d43645b1ac..02b6f8ea7a6134d6931c2d6da22e20166d139699 100644 --- a/src/main/java/com/huawei/unt/UNTMain.java +++ b/src/main/java/com/huawei/unt/UNTMain.java @@ -41,6 +41,7 @@ public class UNTMain { String basePath = args[2]; String jarPathHash = TranslatorUtils.getJarHashPath(jarPath); + LOGGER.info(String.format("translate logs for jar %s start", jarPathHash)); EngineType engine = TranslatorUtils.getEngineType(engineType); @@ -120,11 +121,31 @@ public class UNTMain { } + String shaFile = basePath + File.separator + SHA256; + + try (FileWriter fileWriter = new FileWriter(shaFile, false)) { + fileWriter.write(jarPathHash); + } catch (IOException e) { + throw new UNTException("Can't create or write file, " + e); + } + + if (!TranslatorContext.MISSING_INTERFACES.isEmpty()) { + StringBuilder errMessage = new StringBuilder(); + errMessage.append("the following methods is missing, auto compilation is terminated!\n"); + for (String className : TranslatorContext.MISSING_INTERFACES.keySet()) { + errMessage.append(className).append(": \n"); + for (String method : TranslatorContext.MISSING_INTERFACES.get(className)) { + errMessage.append(TranslatorContext.TAB).append(method).append(TranslatorContext.NEW_LINE); + } + } + LOGGER.error(errMessage.toString()); + throw new TranslatorException(errMessage.toString()); + } + String outputDir = basePath + File.separator + "output"; String compileShell = cppDir + File.separator + "compile_translated_udf.sh"; String udfProperties = outputDir + File.separator + jarPathHash + File.separator + "udf.properties"; String outPath = outputDir + File.separator + jarPathHash + File.separator; - String shaFile = basePath + File.separator + SHA256; try (FileWriter fileWriter = new FileWriter(shaFile, false)) { fileWriter.write(jarPathHash); @@ -153,9 +174,32 @@ public class UNTMain { String mainMakefileTemplate; String subMakefileTemplate; try { - mainMakefileTemplate = new String(Files.readAllBytes(Paths.get(templates, "main-Makefile"))); + + + mainMakefileTemplate = new String(Files.readAllBytes(Paths.get(templates, "main-Makefile"))); subMakefileTemplate = new String(Files.readAllBytes(Paths.get(templates, "sub-Makefile"))); - } catch (IOException e) { + //makefile + if (TranslatorContext.ISREGEXACC){ + if (!TranslatorContext.UDF_MAP.containsKey("regex_lib_path")){ + LOGGER.warn("regex lib path not exist, use default path: /usr/local/ksl/lib/libKHSEL_ops.a"); + }else { + String regexLibPath = TranslatorContext.UDF_MAP.get("regex_lib_path"); + File file = new File(regexLibPath); + if (file.exists() && !file.isDirectory()){ + mainMakefileTemplate = mainMakefileTemplate.replace("regex_lib := /usr/local/ksl/lib/libKHSEL_ops.a", "regex_lib := " + regexLibPath); + }else { + LOGGER.warn("regex lib path not exist, use default path: /usr/local/ksl/lib/libKHSEL_ops.a"); + } + } + }else { + mainMakefileTemplate = mainMakefileTemplate.replace("regex_lib := /usr/local/ksl/lib/libKHSEL_ops.a", ""); + mainMakefileTemplate = mainMakefileTemplate.replace("regex_lib", ""); + subMakefileTemplate = subMakefileTemplate.replace("$(regex_lib)", ""); + } + + mainMakefileTemplate = mainMakefileTemplate.replace("CXXFLAGS := -o3 -std=c++17 -fPIC", "CXXFLAGS := " + TranslatorContext.COMPILEOPTION); + + } catch (IOException e) { throw new TranslatorException("load Makefile templates failed: " + e.getMessage()); } @@ -224,9 +268,9 @@ public class UNTMain { compileCommand = compileCommand + " " + allCppFile; } compileCommand += " " + basicLib + " " + regexLib + " -o " - + outputDir + File.separator + jarPathHash + File.separator + soFileName + TranslatorContext.NEW_LINE; + + outputDir + File.separator + jarPathHash + File.separator + soFileName + ".so" + TranslatorContext.NEW_LINE; - String propertyLine = classes.get(i).getClassName() + "=" + soFileName + TranslatorContext.NEW_LINE; + String propertyLine = classes.get(i).getClassName() + "=" + soFileName + ".so" + TranslatorContext.NEW_LINE; compileShellWriter.write(compileCommand); udfPropertiesWriter.write(propertyLine); } diff --git a/src/main/java/com/huawei/unt/dependency/DependencyAnalyzer.java b/src/main/java/com/huawei/unt/dependency/DependencyAnalyzer.java index e1fb59007c33404abfbfea47734a7637fc8ddd23..30fabd0185b196723c203d95c79ebe616ab2ab86 100644 --- a/src/main/java/com/huawei/unt/dependency/DependencyAnalyzer.java +++ b/src/main/java/com/huawei/unt/dependency/DependencyAnalyzer.java @@ -91,19 +91,13 @@ public class DependencyAnalyzer { JavaClass javaClass = addJsonConstructorQueue.poll(); boolean hasObjectField = false; + HashSet needToJsonConstruct = new HashSet<>(); + Set supperClasses = javaClass.getSupperClasses().stream() .filter(c -> !TranslatorContext.CLASS_MAP.containsKey(c.getFullyQualifiedName())) .collect(Collectors.toSet()); - for (ClassType supperClass : supperClasses) { - try{ - JavaClass supperJavaClass = allClasses.get(supperClass); - addJsonConstructorMap.put(JavaIdentifierFactory.getInstance().getClassType(supperClass.getClassName()), supperJavaClass); - addJsonConstructorQueue.add(supperJavaClass); - }catch (Exception e){ - LOGGER.error("supperClass {} not found in allNeedTransClasses ", supperClass.getClassName()); - } - } + needToJsonConstruct.addAll(supperClasses); Set fields = javaClass.getFields(); for (JavaSootField field : fields) { @@ -117,15 +111,24 @@ public class DependencyAnalyzer { if (TranslatorContext.CLASS_MAP.containsKey(fieldClassType.getFullyQualifiedName())){ continue; } - try{ - JavaClass fieldJavaClass = allClasses.get(fieldClassType); - addJsonConstructorMap.put(JavaIdentifierFactory.getInstance().getClassType(fieldJavaClass.getClassName()), fieldJavaClass); - addJsonConstructorQueue.add(fieldJavaClass); - }catch (Exception e){ - LOGGER.error("fieldClassType {} not found in allNeedTransClasses ", fieldClassType.getClassName()); - } + needToJsonConstruct.add(fieldClassType); + } + } + + Set newJsonConstruct = needToJsonConstruct.stream() + .filter(c -> !addJsonConstructorMap.containsKey(c)) + .collect(Collectors.toSet()); + + for (ClassType classType : newJsonConstruct) { + try{ + JavaClass newJsonjavaClass = allClasses.get(classType); + addJsonConstructorMap.put(JavaIdentifierFactory.getInstance().getClassType(newJsonjavaClass.getClassName()), newJsonjavaClass); + addJsonConstructorQueue.add(newJsonjavaClass); + }catch (Exception e){ + LOGGER.error("fieldClassType {} not found in allNeedTransClasses ", classType.getClassName()); } } + if (hasObjectField){ javaClass.setHasObjectField(); } diff --git a/src/main/java/com/huawei/unt/optimizer/ArrayFieldHandler.java b/src/main/java/com/huawei/unt/optimizer/ArrayFieldHandler.java index fecddad61d0f48a76d852d89af8754106feeb854..dfde4eec902f68cffe25114d2e88058102afaa27 100644 --- a/src/main/java/com/huawei/unt/optimizer/ArrayFieldHandler.java +++ b/src/main/java/com/huawei/unt/optimizer/ArrayFieldHandler.java @@ -12,7 +12,6 @@ import sootup.core.jimple.basic.Local; import sootup.core.jimple.basic.Value; import sootup.core.jimple.common.ref.JArrayRef; import sootup.core.jimple.common.stmt.JAssignStmt; -import sootup.core.jimple.common.stmt.JInvokeStmt; import sootup.core.jimple.common.stmt.Stmt; import sootup.core.types.PrimitiveType; import sootup.core.types.Type; diff --git a/src/main/java/com/huawei/unt/optimizer/MemoryReleaseOptimizer.java b/src/main/java/com/huawei/unt/optimizer/MemoryReleaseOptimizer.java index 015645024fca8fe79e72c0decae8cdb0b3ca06a6..0cb1d74796cc78a8494ddd96ab93f8ee760b4839 100644 --- a/src/main/java/com/huawei/unt/optimizer/MemoryReleaseOptimizer.java +++ b/src/main/java/com/huawei/unt/optimizer/MemoryReleaseOptimizer.java @@ -28,12 +28,10 @@ import sootup.core.jimple.common.stmt.JReturnStmt; import sootup.core.jimple.common.stmt.JReturnVoidStmt; import sootup.core.jimple.common.stmt.Stmt; import sootup.core.signatures.MethodSignature; -import sootup.core.signatures.PackageName; import sootup.core.types.ClassType; import sootup.core.types.PrimitiveType; import sootup.core.types.Type; import sootup.core.types.VoidType; -import sootup.java.core.types.JavaClassType; import java.util.ArrayList; import java.util.HashMap; @@ -49,7 +47,6 @@ public class MemoryReleaseOptimizer implements Optimizer { private final static Logger LOGGER = LoggerFactory.getLogger(MemoryReleaseOptimizer.class); public final static Integer BEFORE = 0; public final static Integer AFTER = 1; - private final static JavaClassType ITERATOR = new JavaClassType("Iterator", new PackageName("java.util")); private final Map refMap; private List stmts; @@ -286,13 +283,15 @@ public class MemoryReleaseOptimizer implements Optimizer { Stmt stmt = stmts.get(i); LValue leftOp = ((JAssignStmt) stmt).getLeftOp(); + Value value = ((JAssignStmt) stmt).getRightOp(); + int ref = getRef(value); if (!(leftOp.getType() instanceof PrimitiveType)) { - Value value = ((JAssignStmt) stmt).getRightOp(); - int ref = getRef(value); if (leftOp instanceof Local) { localAssignStmtHandle(i, (Local) leftOp, value, ref); - } else { - refAssignStmtHandle(i, leftOp, value, ref); + } else if (leftOp instanceof JFieldRef){ + fieldRefAssignStmtHandle(i, leftOp, value, ref); + } else if (leftOp instanceof JArrayRef) { + arrRefAssignStmtHandle(i, leftOp, value, ref); } } } @@ -310,15 +309,14 @@ public class MemoryReleaseOptimizer implements Optimizer { updateLocalMap(local, value, ref); } - private void refAssignStmtHandle(int i, LValue leftOp, Value value, int ref) { + private void fieldRefAssignStmtHandle(int i, LValue leftOp, Value value, int ref) { if (!isStaticInit && !isInit) { Map vars = unknownFree.getOrDefault(i, new HashMap<>()); vars.put(leftOp, BEFORE); unknownFree.put(i, vars); } - // todo: check array ref - if (ref == 0 && !(leftOp instanceof JArrayRef)) { + if (ref == 0) { if (value instanceof Local && locals.containsKey(value) && locals.get(value) == 1) { locals.put((Local) value, 0); } else if (value instanceof JCastExpr) { @@ -331,7 +329,12 @@ public class MemoryReleaseOptimizer implements Optimizer { vars.put(leftOp, AFTER); getRef.put(i, vars); } - } else if (ref == 1 && leftOp instanceof JArrayRef) { + } + } + + // deal with arr[0] = new ... arr->append + private void arrRefAssignStmtHandle(int i, LValue leftOp, Value value, int ref) { + if (ref == 1 && leftOp instanceof JArrayRef) { Map vars = unknownFree.getOrDefault(i, new HashMap<>()); vars.put(leftOp, AFTER); unknownFree.put(i, vars); @@ -386,7 +389,6 @@ public class MemoryReleaseOptimizer implements Optimizer { // loop start from i, break to loops.get(i) List continueStmts = new ArrayList<>(); Set blockLocals = new HashSet<>(); - Set blockRefs = new HashSet<>(); int breakTarget = loops.get(i); for (int j = i; j < breakTarget; j++) { @@ -407,7 +409,7 @@ public class MemoryReleaseOptimizer implements Optimizer { continueStmts.add(j); } } else if (stmt instanceof JAssignStmt) { - loopAssignStmtHandle(j, blockLocals, blockRefs); + loopAssignStmtHandle(j, blockLocals); } else if (stmt instanceof JInvokeStmt) { invokeStmtHandle(j); } else if (stmt instanceof JReturnStmt) { @@ -417,17 +419,10 @@ public class MemoryReleaseOptimizer implements Optimizer { } } - // todo: merge continue and break handle? for (Integer end : continueStmts) { freeHandle(end, blockLocals, BEFORE, true); - Set refs = makeNullSet.getOrDefault(end, new HashSet<>()); - refs.addAll(blockRefs); - makeNullSet.put(end, refs); } freeHandle(breakTarget, blockLocals, BEFORE, true); - Set refs = makeNullSet.getOrDefault(breakTarget, new HashSet<>()); - refs.addAll(blockRefs); - makeNullSet.put(breakTarget, refs); for (Local local : blockLocals) { locals.put(local, 0); @@ -436,13 +431,13 @@ public class MemoryReleaseOptimizer implements Optimizer { return breakTarget - 1; } - private void loopAssignStmtHandle(int j, Set blockLocals, Set blockRefs) { + private void loopAssignStmtHandle(int j, Set blockLocals) { Stmt stmt = stmts.get(j); LValue leftOp = ((JAssignStmt) stmt).getLeftOp(); + Value value = ((JAssignStmt) stmt).getRightOp(); + int ref = getRef(value); if (! (leftOp.getType() instanceof PrimitiveType)) { - Value value = ((JAssignStmt) stmt).getRightOp(); - int ref = getRef(value); if (leftOp instanceof Local) { if (localAssign.containsKey(leftOp)) { reassignHelper(j, ref); @@ -453,8 +448,10 @@ public class MemoryReleaseOptimizer implements Optimizer { assignLocation.add(j); localAssign.put((Local) leftOp, assignLocation); } - } else if (! (leftOp instanceof JArrayRef)){ - blockRefs.add(leftOp); + } else if (leftOp instanceof JFieldRef){ + fieldRefAssignStmtHandle(j, leftOp, value, ref); + } else if (leftOp instanceof JArrayRef){ + arrRefAssignStmtHandle(j, leftOp, value, ref); } } } @@ -463,16 +460,17 @@ public class MemoryReleaseOptimizer implements Optimizer { Stmt stmt = stmts.get(i); if (stmt instanceof JInvokeStmt) { Optional expr = ((JInvokeStmt) stmt).getInvokeExpr(); - if (expr.isPresent() && returnObj(expr.get()) && getRef(expr.get()) == 1) { +// if (expr.isPresent() && returnObj(expr.get()) && getRef(expr.get()) == 1) { + if (expr.isPresent() && getRef(expr.get()) == 1) { ignoredReturnValues.add(i); } } } - private boolean returnObj(AbstractInvokeExpr expr) { - Type type = expr.getType(); - return ! (type instanceof PrimitiveType || type instanceof VoidType); - } +// private boolean returnObj(AbstractInvokeExpr expr) { +// Type type = expr.getType(); +// return ! (type instanceof PrimitiveType || type instanceof VoidType); +// } private void returnStmtHandle(int i) { if (finalReturn != -1) { @@ -540,6 +538,7 @@ public class MemoryReleaseOptimizer implements Optimizer { } private int getRef(Value value) { + // todo: check new if (isNewExpr(value) || value instanceof StringConstant || isToString(value)) { return 1; } else if (value instanceof AbstractInvokeExpr) { @@ -560,12 +559,20 @@ public class MemoryReleaseOptimizer implements Optimizer { return refTmp; } } + putMissingInterfaces(methodSignature); + LOGGER.warn(String.format( + "the ref of method %s not found in refMap, use default 0 as ref count", methodSignature)); } - LOGGER.warn(String.format( - "the ref of method %s not found in refMap, use default 0 as ref count", methodSignature)); return 0; } + private void putMissingInterfaces(MethodSignature methodSignature) { + String className = methodSignature.getDeclClassType().getFullyQualifiedName(); + Set missingMethods = TranslatorContext.MISSING_INTERFACES.getOrDefault(className, new HashSet<>()); + missingMethods.add(methodSignature.getSubSignature().toString()); + TranslatorContext.MISSING_INTERFACES.put(methodSignature.getDeclClassType().getFullyQualifiedName(), missingMethods); + } + private final LinkedList superClassQueue = new LinkedList<>(); private final Set searched = new HashSet<>(); diff --git a/src/main/java/com/huawei/unt/optimizer/Optimizers.java b/src/main/java/com/huawei/unt/optimizer/Optimizers.java index 10433e160b304bfaac8a9c85895fd7e385bbc219..4d007e42cab51ceb1eda9a2a6b9e53782f954ead 100644 --- a/src/main/java/com/huawei/unt/optimizer/Optimizers.java +++ b/src/main/java/com/huawei/unt/optimizer/Optimizers.java @@ -13,7 +13,7 @@ public class Optimizers { public static List OPTIMIZERS = ImmutableList.of( new StmtGraphAnalyzer(), new BranchStmtLabeler(), - new ReduceTupleOptimizer(), +// new ReduceTupleOptimizer(), new NewRefOptimizer(), // new RemoveLogger(), new RemoveTrap(), diff --git a/src/main/java/com/huawei/unt/translator/JavaClassTranslator.java b/src/main/java/com/huawei/unt/translator/JavaClassTranslator.java index 3281f6caf310497a2b4cb92f095f1bb428871a08..b12c6a0962baa0ebd764d3eb49f3918b678a5593 100644 --- a/src/main/java/com/huawei/unt/translator/JavaClassTranslator.java +++ b/src/main/java/com/huawei/unt/translator/JavaClassTranslator.java @@ -12,12 +12,7 @@ import sootup.core.types.PrimitiveType; import sootup.java.core.JavaSootField; import sootup.java.core.JavaSootMethod; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.StringJoiner; +import java.util.*; import java.util.stream.Collectors; import static com.huawei.unt.translator.TranslatorContext.NEW_LINE; @@ -92,7 +87,8 @@ public class JavaClassTranslator { StringJoiner joiner = new StringJoiner(", "); for (ClassType superClassType : superClasses) { if (!TranslatorContext.IGNORED_CLASSES.contains(superClassType.getFullyQualifiedName())) { - joiner.add("public " + TranslatorUtils.formatType(superClassType)); + String formatType = TranslatorUtils.formatType(superClassType); + joiner.add("public " + formatType + (TranslatorContext.GENERIC_FUNCTION.contains(formatType) ? "" : "")); } } @@ -411,7 +407,7 @@ public class JavaClassTranslator { .append(TranslatorUtils.formatParamType(sootField.getType())) .append(className) .append("::") - .append(sootField.getName()) + .append(TranslatorUtils.formatFieldName(sootField.getName())) .append(" = nullptr;") .append(NEW_LINE); } diff --git a/src/main/java/com/huawei/unt/translator/JavaMethodTranslator.java b/src/main/java/com/huawei/unt/translator/JavaMethodTranslator.java index ef09928339d5410c5fe4b7f8abb199479aa7947b..2af375abe6c0005ee2a8caea6da5c352df2ec94d 100644 --- a/src/main/java/com/huawei/unt/translator/JavaMethodTranslator.java +++ b/src/main/java/com/huawei/unt/translator/JavaMethodTranslator.java @@ -101,11 +101,6 @@ public class JavaMethodTranslator { valueVisitor.clear(); } - // deal with gotoFree - if (methodContext.hasGoto(i)) { - bodyBuilder.append(TranslatorContext.GOTO_FREE); - } - StringBuilder beforeCodes = new StringBuilder(); StringBuilder afterCodes = new StringBuilder(); @@ -188,6 +183,10 @@ public class JavaMethodTranslator { bodyBuilder.append(TAB).append("callback->process();").append(NEW_LINE); } bodyBuilder.append(afterCodes); + // deal with gotoFree + if (methodContext.hasGoto(i)) { + bodyBuilder.append(TranslatorContext.GOTO_FREE); + } if (methodContext.needRet() && i == methodContext.getFreeLabel()) { bodyBuilder.append(TranslatorContext.RETURN_RET); } @@ -300,12 +299,12 @@ public class JavaMethodTranslator { ((JInvokeStmt) stmt).getInvokeExpr().get().getMethodSignature().getName())) { MethodSignature signature = ((JInvokeStmt) stmt).getInvokeExpr().get().getMethodSignature(); - Type type = ((JInvokeStmt) stmt).getInvokeExpr().get().getType(); + ClassType declClassType = ((JInvokeStmt) stmt).getInvokeExpr().get().getMethodSignature().getDeclClassType(); List args = ((JInvokeStmt) stmt).getInvokeExpr().get().getArgs(); if (signature.getParameterCount() > 0) { - initBuilder.append(" : ").append(TranslatorUtils.formatType(type)).append("(") - .append(TranslatorUtils.paramsToString(signature, args, methodContext)).append(")"); + initBuilder.append(" : ").append(TranslatorUtils.formatType(declClassType)) + .append(TranslatorUtils.paramsToString(signature, args, methodContext)); } } } diff --git a/src/main/java/com/huawei/unt/translator/RefAnalyzer.java b/src/main/java/com/huawei/unt/translator/RefAnalyzer.java index c9aa5301be17e093293da4507b75fbd9f96a1744..2ca6866214a9a9f9b5c302098513add50f7e2215 100644 --- a/src/main/java/com/huawei/unt/translator/RefAnalyzer.java +++ b/src/main/java/com/huawei/unt/translator/RefAnalyzer.java @@ -25,8 +25,8 @@ import java.util.Map; import java.util.Set; public class RefAnalyzer { - private static Map refMap = TranslatorContext.LIB_INTERFACE_REF; - private static Map> subclassMap = TranslatorContext.SUBCLASS_MAP; + private static Map refMap = TranslatorContext.LIB_INTERFACE_REF; + private static Map> subclassMap = TranslatorContext.SUBCLASS_MAP; private static Set abstractMethods = new HashSet<>(); public static void analyse(Collection javaClasses) { @@ -43,7 +43,6 @@ public class RefAnalyzer { private static void analyse(JavaSootMethod method) { String signature = method.getSignature().toString(); Type returnType = method.getReturnType(); - System.out.println(method.getSignature().toString()); if (! method.hasBody()) { abstractMethods.add(method); } else if (returnType instanceof VoidType diff --git a/src/main/java/com/huawei/unt/translator/TranslatorContext.java b/src/main/java/com/huawei/unt/translator/TranslatorContext.java index 3c410b709501893e314b3e4da260fcf7c30d73cf..378817da835a69e1d53e1ce4cf5d02042582eeed 100644 --- a/src/main/java/com/huawei/unt/translator/TranslatorContext.java +++ b/src/main/java/com/huawei/unt/translator/TranslatorContext.java @@ -44,20 +44,28 @@ public class TranslatorContext { public static final String OLD_VAR_ASSIGN = TAB + "old%1$S = %1$s;" + NEW_LINE;; public static final String OLD_VAR_PUT = TAB + "if (old%1$S != nullptr){" + NEW_LINE + TAB + TAB + "old%1$S->putRefCount();" + NEW_LINE + - TAB + "}" + NEW_LINE;; - + TAB + "}" + NEW_LINE; + // todo + public static int TUNELEVEL; + public static boolean ISMEMTUNE; + public static boolean ISHWACCTUNE; + public static boolean ISREGEXACC; +// public static String REGEXLIBPATH; + public static String COMPILEOPTION; + public static Map> SUPERCLASS_MAP = new HashMap<>(); + public static Map> SUBCLASS_MAP = new HashMap<>(); + public static Map> MISSING_INTERFACES = new HashMap<>(); public static Map UDF_MAP; public static Map CLASS_MAP; public static Map INCLUDE_MAP; public static Map FUNCTION_MAP; - public static Map> SUPERCLASS_MAP; - public static Map> SUBCLASS_MAP; public static Map LIB_INTERFACE_REF; public static Set FILTER_PACKAGES; public static Set IGNORED_CLASSES; public static Set IGNORED_METHODS; public static Set PRIMARY_TYPES; public static Set STD_STRING_METHODS; + public static Set GENERIC_FUNCTION; // todo: make it configurable public static final int MAX_CLASS_DEPTH = 100; @@ -66,6 +74,15 @@ public class TranslatorContext { public static void init(String configDir) { // make it log info + GENERIC_FUNCTION = new HashSet<>(); + GENERIC_FUNCTION.add("FilterFunction"); + GENERIC_FUNCTION.add("RichParallelSourceFunction"); + GENERIC_FUNCTION.add("FlatMapFunction"); + GENERIC_FUNCTION.add("RichFlatMapFunction"); + GENERIC_FUNCTION.add("MapFunction"); + GENERIC_FUNCTION.add("ReduceFunction"); + GENERIC_FUNCTION.add("KeySelect"); + LOGGER.info("Init TranslatorContext"); String udfConfigDir = (configDir.endsWith(File.separator) ? configDir : configDir + File.separator) @@ -84,14 +101,59 @@ public class TranslatorContext { } catch (IOException e) { throw new TranslatorException("Load udf_config files failed: " + e.getMessage()); } + if (!UDF_MAP.containsKey("base.dir")){ - throw new TranslatorException("base.dir is nt set"); + throw new TranslatorException("base.dir is not set"); + } + if (!UDF_MAP.containsKey("tune_level")){ + LOGGER.info("tune_level is not configured, use default tune_level 0"); + TUNELEVEL = 0; + }else { + int tuneLevel = Integer.valueOf(UDF_MAP.get("tune_level")); + if (tuneLevel > 4 || tuneLevel < 0){ + LOGGER.info("tune_level is incorrectly configured, use default tune_level 0"); + TUNELEVEL = 0; + }else { + LOGGER.info("using tune_level " + tuneLevel); + TUNELEVEL = tuneLevel; + } + } + if ((TUNELEVEL & 2) != 0){ + ISMEMTUNE = true; + LOGGER.info("Enabling Memory Optimization"); + }else { + ISMEMTUNE = false; + LOGGER.info("Use the default memory policy."); + } + + if ((TUNELEVEL & 1) != 0){ + ISHWACCTUNE = true; + LOGGER.info("Enabling Hardware Acceleration Optimization"); + }else { + ISHWACCTUNE = false; + LOGGER.info("Disabling Hardware Acceleration Optimization."); } + + if (ISHWACCTUNE && UDF_MAP.getOrDefault("regex_lib_type","0").equals("1")){ + ISREGEXACC = true; + LOGGER.info("Enabling Regex acc."); + }else { + ISREGEXACC = false; + LOGGER.info("Disabling Regex acc."); + } + + if (!UDF_MAP.containsKey("compile_option")||UDF_MAP.get("compile_option").equals("")){ + COMPILEOPTION = "-o3 -std=c++17 -fPIC"; + LOGGER.info("use default compile_option: -o3 -std=c++17 -fPIC"); + }else { + COMPILEOPTION = UDF_MAP.get("compile_option"); + LOGGER.info("use compile_option: " + COMPILEOPTION); + } + configDir = UDF_MAP.get("base.dir").endsWith(File.separator)?UDF_MAP.get("base.dir") : UDF_MAP.get("base.dir") + File.separator; LOGGER.info("load conf base"); - String classProfile = configDir + "conf" + File.separator + "class.properties"; String functionProfile = configDir + "conf" + File.separator + "function.properties"; String includeProfile = configDir + "conf" + File.separator + "include.properties"; @@ -114,6 +176,7 @@ public class TranslatorContext { try { classProperties.load(Files.newInputStream(Paths.get(classProfile))); + functionProperties.load((Files.newInputStream(Paths.get(functionProfile)))); includeProperties.load((Files.newInputStream(Paths.get(includeProfile)))); @@ -179,10 +242,6 @@ public class TranslatorContext { INCLUDE_MAP.put((String) key, includeProperties.getProperty((String) key)); } - // todo: location - SUPERCLASS_MAP = new HashMap<>(); - SUBCLASS_MAP = new HashMap<>(); - LOGGER.info("load package filter config:"); FILTER_PACKAGES = filterPackages; for (String p : filterPackages) { diff --git a/src/main/java/com/huawei/unt/translator/TranslatorUtils.java b/src/main/java/com/huawei/unt/translator/TranslatorUtils.java index a52b16c96d28aac75b3a1cf913b0767f7e445d5b..6d28f2176a1fac7bd53bae848cae73a9d1ce826e 100644 --- a/src/main/java/com/huawei/unt/translator/TranslatorUtils.java +++ b/src/main/java/com/huawei/unt/translator/TranslatorUtils.java @@ -328,10 +328,10 @@ public class TranslatorUtils { } private static final String ARRAY_TYPE = "Array"; - private static final String ARRAY_HEAD = "basictypes/functions/javautils/Arrays.h"; + private static final String ARRAY_HEAD = "basictypes/Arrays.h"; private static final String JSON_HEAD = "nlohmann/json.hpp"; - private static final String CLASSCONSTANT_HEAD = "basictypes/functions/types/ClassConstant.h"; - private static final String STRINGCONSTANT_HEAD = "basictypes/functions/types/StringConstant.h"; + private static final String CLASSCONSTANT_HEAD = "basictypes/ClassConstant.h"; + private static final String STRINGCONSTANT_HEAD = "basictypes/StringConstant.h"; public static String formatFunctionType(Type type) { if (type instanceof VoidType) { diff --git a/src/main/java/com/huawei/unt/translator/visitor/TranslatorStmtVisitor.java b/src/main/java/com/huawei/unt/translator/visitor/TranslatorStmtVisitor.java index fb2efb3528578c088e19622ff20455e1bb31c43d..223cf572c8fe15ea62a1b5542b2b3c7a7f51a0d1 100644 --- a/src/main/java/com/huawei/unt/translator/visitor/TranslatorStmtVisitor.java +++ b/src/main/java/com/huawei/unt/translator/visitor/TranslatorStmtVisitor.java @@ -3,10 +3,14 @@ package com.huawei.unt.translator.visitor; import com.huawei.unt.optimizer.stmts.OptimizedDirectStmt; import com.huawei.unt.optimizer.stmts.OptimizedJAssignStmt; import com.huawei.unt.optimizer.stmts.OptimizedLinesStmt; +import com.huawei.unt.translator.TranslatorContext; import com.huawei.unt.translator.TranslatorException; import com.huawei.unt.model.MethodContext; +import sootup.core.jimple.basic.LValue; import sootup.core.jimple.basic.Local; +import sootup.core.jimple.basic.Value; import sootup.core.jimple.common.constant.IntConstant; +import sootup.core.jimple.common.ref.JArrayRef; import sootup.core.jimple.common.ref.JCaughtExceptionRef; import sootup.core.jimple.common.ref.JParameterRef; import sootup.core.jimple.common.ref.JThisRef; diff --git a/src/main/java/com/huawei/unt/translator/visitor/TranslatorTypeVisitor.java b/src/main/java/com/huawei/unt/translator/visitor/TranslatorTypeVisitor.java index ee8f8edc3fa714da5610958e45e54e8bbe79a2a5..824ab4a9e152333779a001e4f94a85cc86eb00d6 100644 --- a/src/main/java/com/huawei/unt/translator/visitor/TranslatorTypeVisitor.java +++ b/src/main/java/com/huawei/unt/translator/visitor/TranslatorTypeVisitor.java @@ -1,15 +1,18 @@ package com.huawei.unt.translator.visitor; +import com.huawei.unt.translator.TranslatorContext; import com.huawei.unt.translator.TranslatorException; import com.huawei.unt.translator.TranslatorUtils; import sootup.core.jimple.visitor.AbstractTypeVisitor; +import sootup.core.types.ArrayType; import sootup.core.types.ClassType; +import sootup.core.types.PrimitiveType; import sootup.core.types.Type; import javax.annotation.Nonnull; public class TranslatorTypeVisitor extends AbstractTypeVisitor { - protected final StringBuilder typeBuilder = new StringBuilder(); + private final StringBuilder typeBuilder = new StringBuilder(); private static final TranslatorTypeVisitor INSTANCE = new TranslatorTypeVisitor(); @@ -17,6 +20,7 @@ public class TranslatorTypeVisitor extends AbstractTypeVisitor { type.accept(INSTANCE); String typeString = INSTANCE.toCode(); INSTANCE.clear(); + return typeString; } @@ -62,7 +66,7 @@ public class TranslatorTypeVisitor extends AbstractTypeVisitor { @Override public void caseArrayType() { - typeBuilder.append("Array *"); + typeBuilder.append("Array"); // throw new DependencyException("Array type need special handle now."); } @@ -91,11 +95,11 @@ public class TranslatorTypeVisitor extends AbstractTypeVisitor { throw new TranslatorException("Has unsupported type"); } - public void clear() { + private void clear() { typeBuilder.delete(0, typeBuilder.length()); } - public String toCode() { + private String toCode() { return typeBuilder.toString(); } diff --git a/src/main/java/com/huawei/unt/translator/visitor/TranslatorValueVisitor.java b/src/main/java/com/huawei/unt/translator/visitor/TranslatorValueVisitor.java index aa0f05f8944465ba8596504fe4d025215641f937..491410bfcf019844f0e7a01aff9a523b8b74dd19 100644 --- a/src/main/java/com/huawei/unt/translator/visitor/TranslatorValueVisitor.java +++ b/src/main/java/com/huawei/unt/translator/visitor/TranslatorValueVisitor.java @@ -1,6 +1,7 @@ package com.huawei.unt.translator.visitor; import com.huawei.unt.optimizer.stmts.OptimizedValue; +import com.huawei.unt.translator.TranslatorContext; import com.huawei.unt.translator.TranslatorException; import com.huawei.unt.translator.TranslatorUtils; import com.huawei.unt.model.MethodContext; @@ -60,6 +61,7 @@ import sootup.core.jimple.common.ref.JThisRef; import sootup.core.jimple.common.ref.Ref; import sootup.core.jimple.visitor.AbstractValueVisitor; import sootup.core.types.PrimitiveType; +import sootup.core.types.Type; import javax.annotation.Nonnull; @@ -455,7 +457,14 @@ public class TranslatorValueVisitor extends AbstractValueVisitor { valueBuilder.append(base); valueBuilder.append("->"); - valueBuilder.append(expr.getMethodSignature().getName()) + String methodName = expr.getMethodSignature().getName(); + + if (expr.getMethodSignature().toString().equals("") + && TranslatorContext.UDF_MAP.getOrDefault("regex_lib_type","0").equals("1")){ + methodName += "_tune"; + } + + valueBuilder.append(methodName) .append(TranslatorUtils.paramsToString(expr.getMethodSignature(), expr.getArgs(), methodContext)); } @@ -484,7 +493,10 @@ public class TranslatorValueVisitor extends AbstractValueVisitor { public void caseStaticInvokeExpr(@Nonnull JStaticInvokeExpr expr) { String staticFunction = TranslatorTypeVisitor.getTypeString(expr.getMethodSignature().getDeclClassType()) + "::" + expr.getMethodSignature().getName(); - + if (TranslatorContext.ISMEMTUNE && + staticFunction.equals("Long::valueOf")){ + staticFunction = staticFunction + "_tune"; + } valueBuilder.append(TranslatorUtils.formatFunctionName(staticFunction)) .append(TranslatorUtils.paramsToString(expr.getMethodSignature(), expr.getArgs(), methodContext)); } @@ -517,7 +529,6 @@ public class TranslatorValueVisitor extends AbstractValueVisitor { super.casePhiExpr(expr); } - // todo: check translated Array type @Override public void caseNewArrayExpr(@Nonnull JNewArrayExpr expr) { TranslatorValueVisitor visitor = new TranslatorValueVisitor(methodContext); @@ -528,7 +539,7 @@ public class TranslatorValueVisitor extends AbstractValueVisitor { @Override public void caseNewMultiArrayExpr(@Nonnull JNewMultiArrayExpr expr) { - // todo : fill it + //todo: fill multiArr throw new TranslatorException("NewMultiArrayExpr is not supported now."); } diff --git a/src/main/java/com/huawei/unt/type/flink/FlinkFlatMapFunction.java b/src/main/java/com/huawei/unt/type/flink/FlinkFlatMapFunction.java index e24899331b4210c99d7d19b55b146e31f8e877b0..89f87e9e67f67538f6c434996704465b2cf7034f 100644 --- a/src/main/java/com/huawei/unt/type/flink/FlinkFlatMapFunction.java +++ b/src/main/java/com/huawei/unt/type/flink/FlinkFlatMapFunction.java @@ -30,7 +30,7 @@ public class FlinkFlatMapFunction implements UDFType { @Override public String getCppFileString(String className) { return "#include \"../" + className + ".h\"\n\n" - + "extern \"C\" std::unique_ptr NewInstance(nlohmann::json jsonObj) {\n" + + "extern \"C\" std::unique_ptr> NewInstance(nlohmann::json jsonObj) {\n" + " return std::make_unique<" + className + ">(jsonObj);\n" + "}"; } diff --git a/src/main/java/com/huawei/unt/type/flink/FlinkKeySelector.java b/src/main/java/com/huawei/unt/type/flink/FlinkKeySelector.java index f0cd1a17551c37b2e54e60eb5234329279fff1b7..9506ae1b01d3f9d749de1b3fa470056a383474ba 100644 --- a/src/main/java/com/huawei/unt/type/flink/FlinkKeySelector.java +++ b/src/main/java/com/huawei/unt/type/flink/FlinkKeySelector.java @@ -32,7 +32,7 @@ public class FlinkKeySelector implements UDFType { public String getCppFileString(String className) { return "#include \"../" + className + ".h\"\n\n" - + "extern \"C\" std::unique_ptr NewInstance(nlohmann::json jsonObj) {\n" + + "extern \"C\" std::unique_ptr> NewInstance(nlohmann::json jsonObj) {\n" + " return std::make_unique<" + className + ">(jsonObj);\n" + "}"; diff --git a/src/main/java/com/huawei/unt/type/flink/FlinkMapFunction.java b/src/main/java/com/huawei/unt/type/flink/FlinkMapFunction.java index db0e951d02cfebeacba226f4649da9449b0dd114..87174de866af434fac7d5bd529ee9fa492e52a44 100644 --- a/src/main/java/com/huawei/unt/type/flink/FlinkMapFunction.java +++ b/src/main/java/com/huawei/unt/type/flink/FlinkMapFunction.java @@ -28,7 +28,7 @@ public class FlinkMapFunction implements UDFType { @Override public String getCppFileString(String className) { return "#include \"../" + className + ".h\"\n\n" - + "extern \"C\" std::unique_ptr NewInstance(nlohmann::json jsonObj) {\n" + + "extern \"C\" std::unique_ptr> NewInstance(nlohmann::json jsonObj) {\n" + " return std::make_unique<" + className + ">(jsonObj);\n" + "}"; } diff --git a/src/main/java/com/huawei/unt/type/flink/FlinkReduceFunction.java b/src/main/java/com/huawei/unt/type/flink/FlinkReduceFunction.java index cbda5019a28a2448229ac29e246ce475469dcbb2..2bf62195f79973946a0207954618fd7478f1dcd9 100644 --- a/src/main/java/com/huawei/unt/type/flink/FlinkReduceFunction.java +++ b/src/main/java/com/huawei/unt/type/flink/FlinkReduceFunction.java @@ -29,7 +29,7 @@ public class FlinkReduceFunction implements UDFType { @Override public String getCppFileString(String className) { return "#include \"../" + className + ".h\"\n" - + "extern \"C\" std::unique_ptr NewInstance(nlohmann::json jsonObj) {\n" + + "extern \"C\" std::unique_ptr> NewInstance(nlohmann::json jsonObj) {\n" + " return std::make_unique<" + className + ">(jsonObj);\n" + "}"; } diff --git a/src/main/java/com/huawei/unt/type/flink/FlinkRichFilterFunction.java b/src/main/java/com/huawei/unt/type/flink/FlinkRichFilterFunction.java index 2e390621221d84c115bcbe3c962b08602e6bcd38..fed0fa4b69e42fc3d1827211a8ee8f827ffae4f8 100644 --- a/src/main/java/com/huawei/unt/type/flink/FlinkRichFilterFunction.java +++ b/src/main/java/com/huawei/unt/type/flink/FlinkRichFilterFunction.java @@ -30,7 +30,7 @@ public class FlinkRichFilterFunction implements UDFType { @Override public String getCppFileString(String className) { return "#include \"../" + className + ".h\"" + NEW_LINE + NEW_LINE - + "extern \"C\" std::unique_ptr NewInstance(nlohmann::json jsonObj) {" + NEW_LINE + + "extern \"C\" std::unique_ptr> NewInstance(nlohmann::json jsonObj) {" + NEW_LINE + " return std::make_unique<" + className + ">(jsonObj);" + NEW_LINE + "}"; } diff --git a/src/main/java/com/huawei/unt/type/flink/FlinkRichFlatMapFunction.java b/src/main/java/com/huawei/unt/type/flink/FlinkRichFlatMapFunction.java index 4195c9d50deb57c73aa3c6dca72bf5fdb9865326..f806a8576784da04942ccd3f5ae843eca60ce20a 100644 --- a/src/main/java/com/huawei/unt/type/flink/FlinkRichFlatMapFunction.java +++ b/src/main/java/com/huawei/unt/type/flink/FlinkRichFlatMapFunction.java @@ -31,7 +31,7 @@ public class FlinkRichFlatMapFunction implements UDFType { @Override public String getCppFileString(String className) { return "#include \"../" + className + ".h\"\n\n" - + "extern \"C\" std::unique_ptr NewInstance(nlohmann::json jsonObj) {\n" + + "extern \"C\" std::unique_ptr> NewInstance(nlohmann::json jsonObj) {\n" + " return std::make_unique<" + className + ">(jsonObj);\n" + "}"; } diff --git a/src/main/java/com/huawei/unt/type/flink/FlinkRichParallelSourceFunction.java b/src/main/java/com/huawei/unt/type/flink/FlinkRichParallelSourceFunction.java index 4e297484248b9c8d1c79066026c535b68bfe14ba..51f3d8a62e09851801e848978d883cd134790a68 100644 --- a/src/main/java/com/huawei/unt/type/flink/FlinkRichParallelSourceFunction.java +++ b/src/main/java/com/huawei/unt/type/flink/FlinkRichParallelSourceFunction.java @@ -31,7 +31,7 @@ public class FlinkRichParallelSourceFunction implements UDFType { public String getCppFileString(String className) { return "#include \"../" + className + ".h\"\n\n" - + "extern \"C\" std::unique_ptr NewInstance(nlohmann::json jsonObj) {\n" + + "extern \"C\" std::unique_ptr> NewInstance(nlohmann::json jsonObj) {\n" + " return std::make_unique<" + className + ">(jsonObj);\n" + "}"; } diff --git a/src/main/resources/bin/locked_exec.py b/src/main/resources/bin/locked_exec.py new file mode 100644 index 0000000000000000000000000000000000000000..6ced4be6fca0e33588306e5276b81fcce15dc7b0 --- /dev/null +++ b/src/main/resources/bin/locked_exec.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import sys +import io +import os +import fcntl +import subprocess + +MIN_PYTHON = (3, 6) +if sys.version_info < MIN_PYTHON: + sys.exit( + f"Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]}+ version require\n" + f"current version{sys.version.split()[0]}" + ) + +sys.stdin = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') +sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', write_through=True) +sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', write_through=True) + +LOCK_FILE = os.path.join(os.getcwd(), 'script_temp.lock') + +def main(): + + with open(LOCK_FILE, 'a+') as f: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + + if len(sys.argv) < 2: + print("Error: none command to be executed") + sys.exit(1) + + command = ' '.join(sys.argv[1:]) + try: + subprocess.run(command, shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"exec command failederrorcode: {e.returncode}") + sys.exit(e.returncode) + +if __name__ == "__main__": + main() diff --git a/src/main/resources/bin/udf_translate.sh b/src/main/resources/bin/udf_translate.sh index 7ac0ac50257bb2201a42c745b82190a31b2e5801..1ece755006db26a4c8f4e9100237f27878e9e61b 100644 --- a/src/main/resources/bin/udf_translate.sh +++ b/src/main/resources/bin/udf_translate.sh @@ -19,7 +19,7 @@ if [ ! -d "$cppDir" ]; then mkdir "$cppDir" fi -java -Dlog4j.configurationFile="${baseDir}/conf/log4j2.xml" -cp "${baseDir}/lib/*" com.huawei.unt.UNTMain "${jarPath}" "${engineType}" "${baseDir}" +python locked_exec.py java -Dlog4j.configurationFile="\"${baseDir}/conf/log4j2.xml\"" -cp "\"${baseDir}/lib/*\"" com.huawei.unt.UNTMain "${jarPath}" "${engineType}" "${baseDir}" export C_INCLUDE_PATH=${cppDir}/include:$C_INCLUDE_PATH export CPLUS_INCLUDE_PATH=${cppDir}/include:$CPLUS_INCLUDE_PATH diff --git a/src/main/resources/conf/unt_conf.properties b/src/main/resources/conf/unt_conf.properties index 3b37bffc9ce4ae8733b4cffbc2d096777a820e0f..2810ac0705a8711f31c0687d8fb944d618476321 100644 --- a/src/main/resources/conf/unt_conf.properties +++ b/src/main/resources/conf/unt_conf.properties @@ -1,2 +1,8 @@ base.dir=/opt/libbasictypes -#ai4compile=/opt +tune_level=2 +basic_lib_path=/opt/udf-trans-opt/libbasictypes/lib +array_lib_type=0 +regex_lib_type=1 +compile_option= + + diff --git a/src/main/resources/templates/main-Makefile b/src/main/resources/templates/main-Makefile index a951abe7f607313bdb5cf01c5ee82203029eae3d..b762d5d344fb9666d165bc78393687d5f92f15d0 100644 --- a/src/main/resources/templates/main-Makefile +++ b/src/main/resources/templates/main-Makefile @@ -1,7 +1,10 @@ basic_dir := ${basicPath} basic_inc := $(basic_dir)include basic_lib := $(basic_dir)lib/libbasictypes.a -regex_lib := $(basic_dir)lib/libKHSEL_ops.a +function_lib := $(basic_dir)lib/libbasicfunctions.a +third_lib := $(basic_dir)lib/libthirdlibrary.a +json_lib := $(basic_dir)lib/libkaccgson.a +regex_lib := /usr/local/ksl/lib/libKHSEL_ops.a CXX := g++ CXXFLAGS := -o3 -std=c++17 -fPIC -I$(basic_inc) @@ -10,7 +13,7 @@ LDFLAGS := -shared SRC_DIR := $(CURDIR) OUTPUT_DIR := $(SRC_DIR)/../../output/${sha256} -export SRC_DIR OUTPUT_DIR CXX CXXFLAGS LDFLAGS basic_lib regex_lib +export SRC_DIR OUTPUT_DIR CXX CXXFLAGS LDFLAGS basic_lib function_lib third_lib json_lib regex_lib SUBDIRS := $(shell find . -mindepth 2 -name Makefile -exec dirname {} \; | sort | uniq) diff --git a/src/main/resources/templates/sub-Makefile b/src/main/resources/templates/sub-Makefile index ae000126cebdf812f570830f66afc1be5228a24d..c5f4ffc3461f39fe9d3c67d840495f43bf9c874a 100644 --- a/src/main/resources/templates/sub-Makefile +++ b/src/main/resources/templates/sub-Makefile @@ -5,7 +5,7 @@ OBJS := $(SRCS:.cpp=.o) TARGET := $(OUTPUT_DIR)/$(lib_name).so all: $(TARGET) -$(TARGET):$(OBJS) $(basic_lib) $(regex_lib) +$(TARGET):$(OBJS) $(json_lib) $(third_lib) $(function_lib) $(basic_lib) $(regex_lib) @mkdir -p $(@D) $(CXX) $(LDFLAGS) -o $@ $^ @echo "Built $@"