diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 000000000..35eb1ddfb
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/examples/naive.boa b/examples/naive.boa
new file mode 100644
index 000000000..53d78f143
--- /dev/null
+++ b/examples/naive.boa
@@ -0,0 +1,172 @@
+p: Project = input;
+type fv = {a:int, b:int, c:int, d:int};
+type stats = {a_stat:float, b_stat:float, c_stat:float};
+type complete_stat = {avg: stats, dev: stats};
+type Data = {training: fv, testing: fv};
+splitRatio : float = 0.67;
+
+naive := function(vals : array of Data) : float {
+ train : array of fv;
+ test : array of fv;
+
+ spearated: map[int] of array of fv; # classified per value
+ summaries : map[int] of complete_stat;
+
+ # separate the training and testing datasets
+ foreach(i:int; def(vals[i])) {
+ if(def(train)) {
+ train = train + {vals[i].training};
+ } else {
+ train = {vals[i].training};
+ }
+ if(def(test)) {
+ test = test+ {vals[i].testing};
+ } else {
+ test = {vals[i].testing};
+ }
+
+ }
+
+
+ # classify training datasets
+ foreach(i:int; def(train[i])) {
+ temp : array of fv = {train[i]};
+ if(!haskey(spearated, train[i].d)) {
+ spearated[train[i].d] = temp;
+ } else {
+ spearated[train[i].d] = spearated[train[i].d] + temp;
+ }
+ }
+
+ # all the classes
+ classes : array of int = keys(spearated);
+
+ # summarize data from training dataset
+ foreach(i:int; def(classes[i])) {
+ # calculate mean
+ feature_mean : stats = {0.0, 0.0, 0.0};
+ foreach(j:int; def(spearated[classes[i]][j])) {
+ feature_mean.a_stat = feature_mean.a_stat + spearated[classes[i]][j].a;
+ feature_mean.b_stat = feature_mean.b_stat + spearated[classes[i]][j].b;
+ feature_mean.c_stat = feature_mean.c_stat + spearated[classes[i]][j].c;
+ }
+ feature_mean.a_stat = feature_mean.a_stat / len(spearated[classes[i]]);
+ feature_mean.b_stat = feature_mean.b_stat / len(spearated[classes[i]]);
+ feature_mean.c_stat = feature_mean.c_stat / len(spearated[classes[i]]);
+
+
+ # calculate sd
+ feature_sd : stats = {0.0, 0.0, 0.0};
+ foreach(j:int; def(spearated[classes[i]][j])) {
+ feature_sd.a_stat = feature_sd.a_stat + (spearated[classes[i]][j].a - feature_mean.a_stat);
+ feature_sd.b_stat = feature_sd.b_stat + (spearated[classes[i]][j].b - feature_mean.b_stat);
+ feature_sd.c_stat = feature_sd.c_stat + (spearated[classes[i]][j].c - feature_mean.c_stat);
+ }
+ feature_sd.a_stat = sqrt(feature_sd.a_stat / len(spearated[classes[i]]));
+ feature_sd.b_stat = sqrt(feature_sd.b_stat / len(spearated[classes[i]]));
+ feature_sd.c_stat = sqrt(feature_sd.c_stat / len(spearated[classes[i]]));
+
+ # summarized a class
+ summaries[classes[i]] = {feature_mean, feature_sd};
+ }
+
+
+ predictions: array of int;
+ predictions = new(predictions, len(test), -1);
+
+ # predict for each test data
+ foreach(i:int; def(test[i])) {
+ probabilities : map[int] of float;
+ foreach(j: int; def(classes[j])) {
+ probabilities[classes[j]] = 1.0;
+ mean := summaries[classes[j]].avg;
+ deviation := summaries[classes[j]].dev;
+ probabilities[classes[j]] = probabilities[classes[j]] * (1/ (sqrt(2 * 3.14) * deviation.a_stat)) * (exp(-1 * ((pow((1.0 * test[i].a) - mean.a_stat, 2))/(2 * pow(deviation.a_stat, 2)))));
+ probabilities[classes[j]] = probabilities[classes[j]] * (1/ (sqrt(2 * 3.14) * deviation.a_stat)) * (exp(-1 * ((pow((1.0 * test[i].b) - mean.b_stat, 2))/(2 * pow(deviation.b_stat, 2)))));
+ probabilities[classes[j]] = probabilities[classes[j]] * (1/ (sqrt(2 * 3.14) * deviation.a_stat)) * (exp(-1 * ((pow((1.0 * test[i].c) - mean.c_stat, 2))/(2 * pow(deviation.c_stat, 2)))));
+ }
+
+ bestProb : float = 0;
+ bestLab : int = -1;
+ foreach(j: int; def(classes[j])) {
+ if ((bestLab == -1) || (bestProb < probabilities[classes[j]])) {
+ bestProb = probabilities[classes[j]];
+ bestLab = classes[j];
+ }
+ }
+ predictions[i] = bestLab;
+ }
+
+ correct : float = 0.0;
+ foreach(i:int; def(test[i])) {
+ if(predictions[i] == test[i].d) {
+ correct = correct + 1.0;
+ }
+ }
+ return correct/len(test) * 100;
+};
+
+scale := function(ast: int, method: int, class: int) : int {
+ total : int = 0;
+ if(ast > 1000) {
+ total++;
+ } if(method > 500) {
+ total++;
+ } if(class > 50) {
+ total++;
+ }
+ return total;
+};
+
+
+naive_bayes : output naive of Data;
+
+# count ast nodes
+
+astCount := 0;
+classCount := 0;
+methodCount := 0;
+visit(p, visitor {
+ # only look at the latest snapshot
+ before n: CodeRepository -> {
+ snapshot := getsnapshot(n);
+ foreach (i: int; def(snapshot[i]))
+ visit(snapshot[i]);
+ stop;
+ }
+ before node: Declaration -> {
+ if (node.kind == TypeKind.CLASS) {
+ classCount++;
+ foreach (i: int; node.methods[i]) {
+ methodCount++;
+ }
+ }
+ }
+ # by default, count all visited nodes
+ before _ -> astCount++;
+ # these nodes are not part of the AST, so do nothing when visiting
+ before Project, ChangedFile -> ;
+});
+
+
+
+dummy : fv = {0, 0, 0, 0};
+nondummy : fv = {astCount, methodCount, classCount, scale(astCount, methodCount, classCount)};
+data1: Data = {nondummy, dummy};
+data2: Data = {dummy, nondummy};
+if(rand() > splitRatio)
+ naive_bayes << data1;
+else
+ naive_bayes << data2;
+
+
+if(rand() > splitRatio)
+ naive_bayes << data1;
+else
+ naive_bayes << data2;
+
+
+if(rand() > splitRatio)
+ naive_bayes << data1;
+else
+ naive_bayes << data2;
diff --git a/lib/gson-2.8.0.jar b/lib/gson-2.8.0.jar
new file mode 100644
index 000000000..1235f6381
Binary files /dev/null and b/lib/gson-2.8.0.jar differ
diff --git a/lib/guava-21.0.jar b/lib/guava-21.0.jar
new file mode 100644
index 000000000..061819594
Binary files /dev/null and b/lib/guava-21.0.jar differ
diff --git a/lib/jama-1.0.3.jar b/lib/jama-1.0.3.jar
new file mode 100755
index 000000000..e4013a727
Binary files /dev/null and b/lib/jama-1.0.3.jar differ
diff --git a/src/antlr/Boa.g b/src/antlr/Boa.g
index da1ef4872..f7aa74c50 100644
--- a/src/antlr/Boa.g
+++ b/src/antlr/Boa.g
@@ -179,7 +179,7 @@ outputType returns [OutputType ast]
locals [int l, int c]
@init { $l = getStartLine(); $c = getStartColumn(); }
@after { $ast.setPositions($l, $c, getEndLine(), getEndColumn()); }
- : OUTPUT (tk=SET { $ast = new OutputType(new Identifier($tk.text)); } | id=identifier { $ast = new OutputType($id.ast); }) (LPAREN el=expressionList RPAREN { $ast.setArgs($el.list); })? (LBRACKET m=component RBRACKET { $ast.addIndice($m.ast); })* OF m=component { $ast.setType($m.ast); } (WEIGHT m=component { $ast.setWeight($m.ast); })? (FORMAT LPAREN el=expressionList RPAREN)?
+ : OUTPUT (tk=SET { $ast = new OutputType(new Identifier($tk.text)); } | id=identifier { $ast = new OutputType($id.ast); }) (LPAREN vl=vardeclList RPAREN { $ast.setParams($vl.list); })? (LPAREN el=expressionList RPAREN { $ast.setArgs($el.list); })? (LBRACKET m=component RBRACKET { $ast.addIndice($m.ast); })* OF m=component { $ast.setType($m.ast); } (WEIGHT m=component { $ast.setWeight($m.ast); })? (FORMAT LPAREN el=expressionList RPAREN)?
;
functionType returns [FunctionType ast]
@@ -429,6 +429,20 @@ expressionList returns [ArrayList list]
| e=expression { $list.add($e.ast); } ({ notifyErrorListeners("error: ',' expected"); } e=expression { $list.add($e.ast); } | COMMA e=expression { $list.add($e.ast); })*
;
+
+useraggParamDeclaration returns [VarDeclStatement ast]
+ locals [int l, int c]
+ @init { $l = getStartLine(); $c = getStartColumn(); }
+ @after { $ast.setPositions($l, $c, getEndLine(), getEndColumn()); }
+ : v=forVariableDeclaration { $ast = $v.ast; }
+ ;
+
+vardeclList returns [ArrayList list]
+ @init { $list = new ArrayList(); }
+ : e=useraggParamDeclaration { $list.add($e.ast); } (COMMA e=useraggParamDeclaration { $list.add($e.ast); })*
+ | e=useraggParamDeclaration { $list.add($e.ast); } ({ notifyErrorListeners("error: ',' expected"); } e=useraggParamDeclaration { $list.add($e.ast); } | COMMA e=useraggParamDeclaration { $list.add($e.ast); })*
+ ;
+
conjunction returns [Conjunction ast]
locals [int l, int c]
@init { $l = getStartLine(); $c = getStartColumn(); }
diff --git a/src/java/boa/BoaEnumInterface.java b/src/java/boa/BoaEnumInterface.java
new file mode 100644
index 000000000..f07341a85
--- /dev/null
+++ b/src/java/boa/BoaEnumInterface.java
@@ -0,0 +1,5 @@
+package boa;
+
+public interface BoaEnumInterface {
+ Object getValue();
+}
\ No newline at end of file
diff --git a/src/java/boa/BoaTup.java b/src/java/boa/BoaTup.java
new file mode 100644
index 000000000..0f9b535fe
--- /dev/null
+++ b/src/java/boa/BoaTup.java
@@ -0,0 +1,14 @@
+package boa;
+
+import java.io.IOException;
+import java.util.Collection;
+
+
+public interface BoaTup {
+ public String[] getValues();
+ public byte[] serialize(Object o) throws IOException;
+ public Object getValue(String f);
+ public String toString();
+ public T[] asArray(T[] type);
+ public String[] getFieldNames();
+}
\ No newline at end of file
diff --git a/src/java/boa/aggregators/Aggregator.java b/src/java/boa/aggregators/Aggregator.java
index 6ad1d637b..c0e092b8c 100644
--- a/src/java/boa/aggregators/Aggregator.java
+++ b/src/java/boa/aggregators/Aggregator.java
@@ -18,10 +18,15 @@
import java.io.IOException;
+import boa.BoaTup;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer.Context;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
import boa.functions.BoaCasts;
import boa.io.EmitKey;
import boa.io.EmitValue;
@@ -38,6 +43,29 @@ public abstract class Aggregator {
private Context context;
private EmitKey key;
private boolean combining;
+ private final static Set inBuiltAggs = new HashSet();
+
+ static {
+ inBuiltAggs.add("sum");
+ inBuiltAggs.add("top");
+ inBuiltAggs.add("maximum");
+ inBuiltAggs.add("minimum");
+ inBuiltAggs.add("max");
+ inBuiltAggs.add("min");
+ inBuiltAggs.add("collection");
+ inBuiltAggs.add("mean");
+ inBuiltAggs.add("median");
+ inBuiltAggs.add("stdev");
+ inBuiltAggs.add("quantile");
+ inBuiltAggs.add("kurtosis");
+ inBuiltAggs.add("histogram");
+ inBuiltAggs.add("graphCSV");
+ inBuiltAggs.add("set");
+ inBuiltAggs.add("bottom");
+ inBuiltAggs.add("skewness");
+ inBuiltAggs.add("confidence");
+ inBuiltAggs.add("variance");
+ }
/**
* Construct an Aggregator.
@@ -93,6 +121,13 @@ public void aggregate(final double data) throws IOException, InterruptedExceptio
this.aggregate(BoaCasts.doubleToString(data), null);
}
+ public void aggregate(final BoaTup data, final String metadata) throws IOException, InterruptedException, FinishedException {
+ }
+
+ public void aggregate(final BoaTup data) throws IOException, InterruptedException, FinishedException {
+ this.aggregate(data, null);
+ }
+
@SuppressWarnings("unchecked")
protected void collect(final String data, final String metadata) throws IOException, InterruptedException {
if (this.combining)
@@ -107,6 +142,22 @@ protected void collect(final String data) throws IOException, InterruptedExcepti
this.collect(data, null);
}
+ protected void collect(final BoaTup data) throws IOException, InterruptedException {
+ this.collect(data.toString(), null);
+ }
+
+ protected void collect(final BoaTup[] data) throws IOException, InterruptedException {
+ this.collect(Arrays.toString(data), null);
+ }
+
+ protected void collect(final double[] data) throws IOException, InterruptedException {
+ this.collect(Arrays.toString(data), null);
+ }
+
+ protected void collect(final long[] data) throws IOException, InterruptedException {
+ this.collect(Arrays.toString(data), null);
+ }
+
@SuppressWarnings("unchecked")
protected void collect(final long data, final String metadata) throws IOException, InterruptedException {
this.collect(BoaCasts.longToString(data), metadata);
@@ -157,4 +208,8 @@ public void setKey(final EmitKey key) {
public EmitKey getKey() {
return this.key;
}
+
+ public final static boolean isUserDefinedAggregator(String name) {
+ return !Aggregator.inBuiltAggs.contains(name);
+ }
}
diff --git a/src/java/boa/aggregators/UserDefinedAggregator.java b/src/java/boa/aggregators/UserDefinedAggregator.java
new file mode 100644
index 000000000..240712189
--- /dev/null
+++ b/src/java/boa/aggregators/UserDefinedAggregator.java
@@ -0,0 +1,25 @@
+package boa.aggregators;
+
+
+import boa.compiler.UserDefinedAggregators;
+import boa.datagen.util.FileIO;
+import com.google.gson.Gson;
+
+import java.io.*;
+
+@AggregatorSpec(name = "UserDefinedAgg", formalParameters = { "any", "any" }, type = "UserDefined", canCombine = false)
+public abstract class UserDefinedAggregator extends Aggregator {
+
+ public void store(Object object) {
+ Gson json = new Gson();
+ File output = new File(UserDefinedAggregators.getFileName());
+ final String dest= output.getAbsolutePath() + "/";
+ output.mkdir();
+ writeAsJSON(object, dest + UserDefinedAggregators.getFileName() + ".model");
+ }
+
+ private void writeAsJSON(Object object, String path) {
+ Gson writer = new Gson();
+ FileIO.writeFileContents(new File(path), writer.toJson(object));
+ }
+}
\ No newline at end of file
diff --git a/src/java/boa/compiler/BoaCompiler.java b/src/java/boa/compiler/BoaCompiler.java
index 2140234d0..34a74599d 100644
--- a/src/java/boa/compiler/BoaCompiler.java
+++ b/src/java/boa/compiler/BoaCompiler.java
@@ -155,6 +155,8 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int
try {
if (!parserErrorListener.hasError) {
+ UserDefinedAggregators.setFileName(f.getName());
+ UserDefinedAggregators.setJobName("Job" + jobName);
new TypeCheckingVisitor().start(p, new SymbolTable());
final TaskClassifyingVisitor simpleVisitor = new TaskClassifyingVisitor();
@@ -241,6 +243,7 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int
st.add("jobs", jobs);
st.add("jobnames", jobnames);
st.add("combineTables", CodeGeneratingVisitor.combineAggregatorStrings);
+ st.add("userDeclAgg", CodeGeneratingVisitor.userAggregatorDeclStrings);
st.add("reduceTables", CodeGeneratingVisitor.reduceAggregatorStrings);
st.add("splitsize", isSimple ? 64 * 1024 * 1024 : 10 * 1024 * 1024);
if(DefaultProperties.localDataPath != null) {
diff --git a/src/java/boa/compiler/SymbolTable.java b/src/java/boa/compiler/SymbolTable.java
index 9fb315fa5..49274068f 100644
--- a/src/java/boa/compiler/SymbolTable.java
+++ b/src/java/boa/compiler/SymbolTable.java
@@ -22,6 +22,7 @@
import java.util.*;
import java.util.Map.Entry;
+import boa.aggregators.UserDefinedAggregator;
import org.scannotation.AnnotationDB;
import boa.aggregators.AggregatorSpec;
@@ -42,7 +43,7 @@ public class SymbolTable {
private static final Map, BoaType> protomap;
private static Map idmap;
private static final Map globals;
- private static FunctionTrie globalFunctions;
+ private static FunctionTrie globalFunctions;
private FunctionTrie functions;
private Map locals;
@@ -90,6 +91,7 @@ public class SymbolTable {
idmap.put("float", new BoaFloat());
idmap.put("time", new BoaTime());
idmap.put("string", new BoaString());
+ idmap.put("tuple", new BoaTuple(new ArrayList()));
final BoaProtoTuple[] dslTupleTypes = {
new ASTRootProtoTuple(),
@@ -435,15 +437,28 @@ else if (aggregators.containsKey(name))
}
public List> getAggregators(final String name, final BoaType type) {
- final List> aggregators = new ArrayList>();
-
- if (type instanceof BoaTuple)
- for (final BoaType subType : ((BoaTuple) type).getTypes())
- aggregators.add(this.getAggregator(name, subType));
- else
- aggregators.add(this.getAggregator(name, type));
+ final List> searchResult = new ArrayList>();
+ if (possibleInBuiltAgg(name)) {
+ if (type instanceof BoaTuple)
+ searchResult.add(this.getAggregator(name, type));
+ else if (type instanceof BoaArray)
+ searchResult.add(this.getAggregator(name, ((BoaArray)type).getType()));
+ else
+ searchResult.add(this.getAggregator(name, type));
+ } else if (this.functions.hasFunction(name)) {
+ searchResult.add(UserDefinedAggregator.class);
+ }
+ return searchResult;
+ }
- return aggregators;
+ private boolean possibleInBuiltAgg(String name) {
+ Set names = aggregators.keySet();
+ for (final String entry: names) {
+ if(entry.contains(name)) {
+ return true;
+ }
+ }
+ return false;
}
private static void importFunction(final Method m) {
@@ -495,7 +510,9 @@ private static void importLibs(final List urls) throws IOException {
boa.functions.BoaSortIntrinsics.class,
boa.functions.BoaSpecialIntrinsics.class,
boa.functions.BoaStringIntrinsics.class,
- boa.functions.BoaTimeIntrinsics.class
+ boa.functions.BoaTimeIntrinsics.class,
+ boa.functions.BoaMatrixIntrinsics.class,
+ boa.functions.BoaMLIntrinsics.class
};
for (final Class c : builtinFuncs)
importFunctions(c);
diff --git a/src/java/boa/compiler/UserDefinedAgg.java b/src/java/boa/compiler/UserDefinedAgg.java
new file mode 100644
index 000000000..1eb7d32f4
--- /dev/null
+++ b/src/java/boa/compiler/UserDefinedAgg.java
@@ -0,0 +1,319 @@
+package boa.compiler;
+
+
+import boa.compiler.ast.statements.VarDeclStatement;
+import boa.compiler.ast.types.AbstractType;
+import org.stringtemplate.v4.ST;
+import org.stringtemplate.v4.STGroup;
+import org.stringtemplate.v4.STGroupFile;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class UserDefinedAgg {
+ private final String userGivenName;
+ private final String code;
+
+ public String getCode() {
+ return code;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ private UserDefinedAgg(String name, String code) {
+ this.userGivenName = name;
+ this.code = code;
+ }
+
+ public String getAggClassName() {
+ return "UserDefined$" + this.userGivenName;
+ }
+
+ public static class Builder {
+ private String funcVarName;
+ private String lambdaName;
+ private String lambdaType;
+ private String lambdaInit;
+ private String lambdaInterface;
+ private String userGivenFuncArg = null; // user given param names
+ private String funcArgType = null; // funcVarName of the compiler generated userGivenParams
+ private String returnType;
+ private boolean isAggregator = false;
+ private List aggregatorOptionParamId = new ArrayList();
+ private List aggregatorOptionParamInitializer = new ArrayList();
+ private List aggregatorOutputParamTypes = new ArrayList();
+ private List userAggOptionVariables = new ArrayList();;
+
+
+ public List getAggregatorOptionParamId() {
+ return aggregatorOptionParamId;
+ }
+
+ public boolean addAggregatorOptionParamId(String code, int index) {
+ return this.aggregatorOptionParamId.add(aggregatorOutputParamTypes.get(index).type.toJavaType() + " " + code + ";\n");
+ }
+
+ public boolean addAggregatorOptionParamInitializer(String initializer) {
+ return this.aggregatorOptionParamInitializer.add(initializer);
+ }
+
+ public void setAggregatorOptionParamType(List params) {
+ for(VarDeclStatement stmt: params) {
+ this.aggregatorOutputParamTypes.add(stmt.getType());
+ }
+ }
+
+ public UserDefinedAgg build(){
+ return new UserDefinedAgg(this.funcVarName, this.generateCode());
+ }
+
+ public Builder userGivenName(String name) {
+ this.funcVarName = name;
+ return this;
+ }
+
+ public Builder setLambdaNameAndType(String name) {
+ String[] details = name.split(" ");
+ this.lambdaType = details[0];
+ this.lambdaName = details[1].substring(0, details[1].length()- 2);
+ return this;
+ }
+
+ public Builder lambdaInit(String init) {
+ this.lambdaInit = getInterfaceInitCode(init);
+ return this;
+ }
+
+ public Builder returnType(String type) {
+ this.returnType = type;
+ return this;
+ }
+
+ public boolean isAggregator() {
+ return this.isAggregator;
+ }
+
+ public boolean isAggregator(boolean flag) {
+ this.isAggregator = flag;
+ return this.isAggregator;
+ }
+
+ public Builder lambdaInterface(String decl) {
+ this.lambdaInterface = updatedInterDeclAndGetOptionVars(decl);
+ return this;
+ }
+
+ //FIXME: This currently just handles only one argument in the user defined aggregators
+ public Builder argTypeName(String param) {
+ if(this.funcArgType == null) {
+ StringBuffer typeNameBuilder = new StringBuffer(param.substring(0, param.length() - 2));
+ typeNameBuilder.setCharAt(0, Character.toUpperCase(typeNameBuilder.charAt(0)));
+ this.funcArgType = typeNameBuilder.toString();
+ }
+ return this;
+ }
+
+ //FIXME: This currently just handles only one argument in the user defined aggregators
+ public Builder userGivenFuncArg(String param) {
+ if(this.userGivenFuncArg == null) {
+ this.userGivenFuncArg = param;
+ }
+ return this;
+ }
+
+ public boolean isParam(String name) {
+ return this.funcArgType.equals(name) || this.funcArgType.equals(name + "[]");
+ }
+
+ public String getFuncVarName() {
+ return this.funcVarName;
+ }
+
+ public String getLambdaType() {
+ return lambdaType;
+ }
+
+ private String getOverridingMethds() {
+ String parser = typeParser(funcArgType.toString());
+ STGroup stg = new STGroupFile("templates/UserAggregatorClass.stg");
+ final ST st = stg.getInstanceOf("OverridenMethods");
+
+ st.add("name", this.lambdaName);
+ st.add("type", funcArgType.toString());
+ st.add("returnType", this.returnType);
+ if(parser.isEmpty()){
+ st.add("nonPrimitive", true);
+ }else {
+ st.add("parser", parser);
+ st.add("javaPrimitiveType", funcArgType.toLowerCase());
+ }
+ return st.render();
+ }
+
+ private String generateCode() {
+ STGroup stg = new STGroupFile("templates/UserAggregatorClass.stg");
+ final ST st = stg.getInstanceOf("AggregatorClass");
+
+ st.add("funcName", this.getFuncVarName());
+ st.add("methods", fulQualifiedNameGne(this.getOverridingMethds()));
+ st.add("lambdaInit", fulQualifiedNameGne(this.lambdaInit));
+ st.add("lambdaType", this.lambdaType);
+ st.add("lambdaName", this.lambdaName);
+ st.add("interface", fulQualifiedNameGne(this.lambdaInterface));
+ st.add("aggParams", removeFinal());
+ if(this.funcArgType != null) {
+ st.add("funcArg", fulQualifiedNameGne(funcArgType));
+ }else {
+ st.add("funcArg", this.userGivenFuncArg);
+ }
+
+ String constructor = getAggregatorOptionalVarInit(userAggOptionVariables);
+ if(!constructor.isEmpty())
+ st.add("constcode", constructor);
+
+ return st.render();
+ }
+
+
+ private String fulQualifiedNameGne(String str) {
+ StringBuffer qualifiedName = new StringBuffer();
+ qualifiedName
+ .append("boa.")
+ .append(UserDefinedAggregators.getFileName())
+ .append(".")
+ .append(UserDefinedAggregators.getFileName())
+ .append("BoaMapper")
+ .append(".")
+ .append(UserDefinedAggregators.getJobName())
+ .append(".BoaTup_");
+ return str.replace("BoaTup_", qualifiedName.toString());
+ }
+
+ private boolean isPrimitive(String typeName) {
+ return "long".equalsIgnoreCase(typeName)
+ || "int".equalsIgnoreCase(typeName)
+ || "long[]".equalsIgnoreCase(typeName)
+ || "int[]".equalsIgnoreCase(typeName)
+ || "integer[]".equalsIgnoreCase(typeName)
+ || "integer".equalsIgnoreCase(typeName)
+ || "float".equalsIgnoreCase(typeName)
+ || "double".equalsIgnoreCase(typeName)
+ || "float[]".equalsIgnoreCase(typeName)
+ || "double[]".equalsIgnoreCase(typeName)
+ || "boolean".equalsIgnoreCase(typeName)
+ || "boolean[]".equalsIgnoreCase(typeName)
+ || "bool".equalsIgnoreCase(typeName)
+ || "bool[]".equalsIgnoreCase(typeName)
+ || "String[]".equalsIgnoreCase(typeName)
+ || "String".equalsIgnoreCase(typeName)
+ || "byte".equalsIgnoreCase(typeName)
+ || "byte[]".equalsIgnoreCase(typeName)
+ || "char[]".equalsIgnoreCase(typeName)
+ || "char".equalsIgnoreCase(typeName);
+ }
+
+ private String typeParser(String typeName) {
+ String result = "";
+ if ("long".equalsIgnoreCase(typeName)
+ || "int".equalsIgnoreCase(typeName)
+ || "long[]".equalsIgnoreCase(typeName)
+ || "int[]".equalsIgnoreCase(typeName)
+ || "integer[]".equalsIgnoreCase(typeName)
+ || "integer".equalsIgnoreCase(typeName)) {
+ result = "Long.parseLong";
+ } else if("float".equalsIgnoreCase(typeName)
+ || "double".equalsIgnoreCase(typeName)
+ || "float[]".equalsIgnoreCase(typeName)
+ || "double[]".equalsIgnoreCase(typeName)) {
+ result = "Double.parseDouble";
+ } else if("boolean".equalsIgnoreCase(typeName)
+ || "boolean[]".equalsIgnoreCase(typeName)
+ || "bool".equalsIgnoreCase(typeName)
+ || "bool[]".equalsIgnoreCase(typeName)) {
+ result = "Boolean.valueOf";
+ }
+ return result;
+ }
+
+ private String updatedInterDeclAndGetOptionVars(String lambdaInterfaceDecl) {
+ final int startindex = lambdaInterfaceDecl.indexOf('(');
+ final int endIndex = lambdaInterfaceDecl.indexOf(')');
+ String allVargs = lambdaInterfaceDecl.substring(startindex + 1, endIndex);
+ String vars[] = allVargs.split(",");
+ if(vars.length > 0) {
+ if(!isArrayArgument(vars[0])) {
+ throw new RuntimeException("First argument of userDefinedAggregator must be array of values");
+ }
+ // update the interface declaration code with removing all of the arguments with just one
+ if(userAggOptionVariables.size() <= 0) {
+ String tmp = vars.length > 1 ? flattenArrayWithSeperator(Arrays.copyOfRange(vars, 1, vars.length), ',') : "";
+ Collections.addAll(userAggOptionVariables, tmp.split(","));
+ }
+ return lambdaInterfaceDecl.replace(allVargs, vars[0]);
+ } else {
+ throw new RuntimeException("Userdefined aggregators must have atleast one argument, which is array of values");
+ }
+ }
+
+ private boolean isArrayArgument(String arg) {
+ return arg.matches("final\\s+.*\\[]\\s+_*.*");
+ }
+
+ private String flattenArrayWithSeperator(final String[] vars, final char seperator) {
+ final StringBuffer flattened = new StringBuffer();
+ for(final String s: vars) {
+ if(s.startsWith("final")) {
+ flattened.append(s.substring(5)).append(seperator);
+ } else {
+ flattened.append(s).append(seperator);
+ }
+ }
+ flattened.deleteCharAt(flattened.length() - 1);
+ return flattened.toString();
+ }
+
+ private String getInterfaceInitCode(String lambdaInterfaceDecl) {
+ final StringBuffer codegenerator = new StringBuffer(lambdaInterfaceDecl);
+ //FIXME: this is an hack in the generated code
+ final int startindex = lambdaInterfaceDecl.indexOf("invoke") + "invoke".length();
+ final int endIndex = lambdaInterfaceDecl.indexOf(" throws"); // space is important here
+ String allVargs = lambdaInterfaceDecl.substring(startindex + 1, endIndex - 1);
+ System.out.println(allVargs);
+ String vars[] = allVargs.split(",");
+ if(vars.length > 0) {
+ return lambdaInterfaceDecl.replace(allVargs, vars[0]);
+ } else {
+ throw new RuntimeException("Userdefined aggregators must have atleast one argument, which is array of values");
+ }
+ }
+
+ private String getAggregatorOptionalVarInit(final List args) {
+ if(args.size() <= 0) {
+ return "";
+ }
+ StringBuffer code = new StringBuffer();
+ StringBuffer param = new StringBuffer();
+ param.append("( ");
+ for(String s: args) {
+ String[] tmp = s.split(" ");
+ code.append("\t\tthis.").append(tmp[tmp.length - 1]).append(" = ").append(tmp[tmp.length - 1]).append(";\n");
+ param.append(s).append(", ");
+ }
+ param.deleteCharAt(param.length() - 2);
+ param.append(" ) {").append("\n\t").append("this();\n\t").append(code.toString()).append("\n}");
+ return param.toString();
+ }
+
+ private List removeFinal() {
+ for(int i = 0; i < userAggOptionVariables.size(); i++) {
+ userAggOptionVariables.set(i, userAggOptionVariables.get(i).substring(7));
+ }
+ return userAggOptionVariables;
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/src/java/boa/compiler/UserDefinedAggregators.java b/src/java/boa/compiler/UserDefinedAggregators.java
new file mode 100644
index 000000000..acf4ce304
--- /dev/null
+++ b/src/java/boa/compiler/UserDefinedAggregators.java
@@ -0,0 +1,69 @@
+package boa.compiler;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import boa.compiler.UserDefinedAgg.Builder;
+
+public class UserDefinedAggregators {
+ private final static List allFunctions = new ArrayList();
+ private static String jobName;
+ private static String fileName;
+
+ public static void setJobName(String jobName) {
+ UserDefinedAggregators.jobName = jobName;
+ }
+
+ public static void filterAggregatorFunctions() {
+ for(int i = 0; i < allFunctions.size(); i++) {
+ if(!allFunctions.get(i).isAggregator()) {
+ allFunctions.remove(i);
+ }
+ }
+ }
+
+ public static void setFileName(String fileName) {
+ fileName = fileName.substring(0, fileName.indexOf('.'));
+ fileName = fileName.substring(0, 1).toUpperCase() + fileName.substring(1);
+ UserDefinedAggregators.fileName = fileName.substring(0, 1).toUpperCase() + fileName.substring(1);;
+ }
+
+ public static Iterator iterator() {
+ return allFunctions.iterator();
+ }
+
+ public static boolean addNewUserFunction(Builder function) {
+ return allFunctions.add(function);
+ }
+
+ public static List getAllFunctions() {
+ return allFunctions;
+ }
+
+ public static Builder findByUserGivenName(String name) {
+ for(Builder function : allFunctions) {
+ if(function.getFuncVarName().equals(name)) {
+ return function;
+ }
+ }
+ return null;
+ }
+
+ public static Builder findByLambdaType(String name) {
+ for(Builder function : allFunctions) {
+ if(function.getLambdaType().equals(name)) {
+ return function;
+ }
+ }
+ return null;
+ }
+
+ public static String getJobName() {
+ return jobName;
+ }
+
+ public static String getFileName() {
+ return fileName;
+ }
+}
\ No newline at end of file
diff --git a/src/java/boa/compiler/ast/types/OutputType.java b/src/java/boa/compiler/ast/types/OutputType.java
index a3e04f766..849c56f9e 100644
--- a/src/java/boa/compiler/ast/types/OutputType.java
+++ b/src/java/boa/compiler/ast/types/OutputType.java
@@ -22,6 +22,7 @@
import boa.compiler.ast.Component;
import boa.compiler.ast.Identifier;
import boa.compiler.ast.expressions.Expression;
+import boa.compiler.ast.statements.VarDeclStatement;
import boa.compiler.visitors.AbstractVisitor;
import boa.compiler.visitors.AbstractVisitorNoArg;
import boa.compiler.visitors.AbstractVisitorNoReturn;
@@ -35,6 +36,7 @@ public class OutputType extends AbstractType {
protected Identifier id;
protected final List args = new ArrayList();
protected final List indices = new ArrayList();
+ protected final List params = new ArrayList();
protected Component t;
protected Component weight;
@@ -72,6 +74,18 @@ public void setArgs(final List args) {
}
}
+ public void setParams(final List params) {
+ this.params.clear();
+ for (final VarDeclStatement e : params) {
+ e.setParent(this);
+ this.params.add(e);
+ }
+ }
+
+ public List getParams() {
+ return params;
+ }
+
public List getIndices() {
return indices;
}
diff --git a/src/java/boa/compiler/visitors/CodeGeneratingVisitor.java b/src/java/boa/compiler/visitors/CodeGeneratingVisitor.java
index f365b6043..ba22d6593 100644
--- a/src/java/boa/compiler/visitors/CodeGeneratingVisitor.java
+++ b/src/java/boa/compiler/visitors/CodeGeneratingVisitor.java
@@ -24,11 +24,14 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import boa.compiler.UserDefinedAgg;
import org.stringtemplate.v4.ST;
+import boa.aggregators.Aggregator;
import boa.aggregators.AggregatorSpec;
import boa.compiler.SymbolTable;
import boa.compiler.TypeCheckException;
+import boa.compiler.UserDefinedAggregators;
import boa.compiler.ast.*;
import boa.compiler.ast.expressions.*;
import boa.compiler.ast.literals.*;
@@ -175,8 +178,15 @@ public void visit(final FixPExpression n) {
/** {@inheritDoc} */
@Override
public void visit(final VarDeclStatement n) {
- if (n.type instanceof BoaTable)
+ if (n.type instanceof BoaTable) {
+ UserDefinedAgg.Builder function = UserDefinedAggregators.findByUserGivenName(((OutputType)n.getType()).getId().getToken());
+ if(function != null) {
+ function.isAggregator(true);
+ function.setAggregatorOptionParamType(((OutputType)n.getType()).getParams());
+ }
return;
+ }
+
final ST st = stg.getInstanceOf("VarDecl");
@@ -187,6 +197,13 @@ public void visit(final VarDeclStatement n) {
st.add("isstatic", true);
code.add(st.render());
+
+ if(n.type instanceof BoaFunction) {
+ UserDefinedAgg.Builder funcTionBuilder = UserDefinedAgg.builder();
+ funcTionBuilder.userGivenName(n.getId().getToken()).setLambdaNameAndType(st.render());
+ funcTionBuilder.returnType(((BoaFunction)n.type).getType().toJavaType());
+ UserDefinedAggregators.addNewUserFunction(funcTionBuilder);
+ }
}
}
@@ -220,9 +237,15 @@ public void visit(final FunctionType n) {
final List args = new ArrayList();
final List types = new ArrayList();
+ UserDefinedAgg.Builder functionBuilder = UserDefinedAggregators.findByLambdaType(name);
+
for (final Component c : params) {
args.add(c.getIdentifier().getToken());
types.add(c.getType().type.toJavaType());
+ if(functionBuilder != null && functionBuilder.isAggregator()) {
+ functionBuilder.argTypeName(c.getType().type.toJavaType());
+ functionBuilder.userGivenFuncArg(c.getIdentifier().getToken());
+ }
}
st.add("name", funcType.toJavaType());
@@ -233,6 +256,10 @@ public void visit(final FunctionType n) {
st.add("args", args);
st.add("types", types);
+ if(functionBuilder != null) {
+ functionBuilder.lambdaInterface(st.render());
+ }
+
code.add(st.render());
}
}
@@ -280,7 +307,6 @@ public void visit(final TupleType n) {
st.add("name", tupType.toJavaType());
st.add("fields", fields);
st.add("types", types);
-
code.add(st.render());
}
}
@@ -505,6 +531,7 @@ public void visit(final Call n) {
final public static List combineAggregatorStrings = new ArrayList();
final public static List reduceAggregatorStrings = new ArrayList();
+ final public static Set userAggregatorDeclStrings = new HashSet();
public CodeGeneratingVisitor(final String name) throws IOException {
this.name = name;
@@ -524,6 +551,7 @@ public void visit(final Program n) {
st.add("name", this.name);
this.varDecl.start(n);
+ UserDefinedAggregators.filterAggregatorFunctions();
this.functionDeclarator.start(n);
this.tupleDeclarator.start(n);
this.enumDeclarator.start(n);
@@ -579,9 +607,30 @@ public void visit(final Program n) {
throw new TypeCheckException(n, e.getMessage(), e);
}
}
+
+ if(Aggregator.isUserDefinedAggregator(entry.getValue().getAggregator())) {
+ UserDefinedAgg function = UserDefinedAggregators
+ .findByUserGivenName(entry.getValue().getAggregator())
+ .build();
+ String gencode = function.getCode();
+ if(!userAggregatorDeclStrings.contains(gencode)) {
+ userAggregatorDeclStrings.add(gencode);
+ }
+ StringBuffer params = new StringBuffer();
+ for(String param: entry.getValue().getParameters()) {
+ params.append(param).append(", ");
+ }
+ params.deleteCharAt(params.length() - 2);
+
+ reduceAggregatorStrings.add("this.aggregators.put(\"" + prefix + "::" + id + "\", new " + function.getAggClassName() + "(" + params.toString() + "));");
+ combines = false;
+ } else {
+ reduceAggregatorStrings.add("this.aggregators.put(\"" + prefix + "::" + id + "\", " + src.toString().substring(2) + ");");
+ }
+
if (combines)
combineAggregatorStrings.add("this.aggregators.put(\"" + prefix + "::" + id + "\", " + src.toString().substring(2) + ");");
- reduceAggregatorStrings.add("this.aggregators.put(\"" + prefix + "::" + id + "\", " + src.toString().substring(2) + ");");
+
}
code.add(st.render());
@@ -798,6 +847,7 @@ public void visit(final Factor n) {
String accept = "";
abortGeneration = false;
+ n.env.setOperandType(n.getOperand().type);
if (!(n.getOp(0) instanceof Call)) {
n.getOperand().accept(this);
n.env.setOperandType(n.getOperand().type);
@@ -1324,6 +1374,23 @@ public void visit(final VarDeclStatement n) {
}
if (type instanceof BoaTable) {
+ UserDefinedAgg.Builder function = UserDefinedAggregators.findByUserGivenName(n.getId().getToken());
+ if(function != null) {
+ n.getInitializer().accept(this);
+ function.lambdaInit(code.removeLast());
+ }
+ if(n.getType() instanceof OutputType) {
+ function = UserDefinedAggregators.findByUserGivenName(((OutputType) n.getType()).getId().getToken());
+ if(function != null) {
+ int counter = 0;
+ for(VarDeclStatement stmt: ((OutputType) n.getType()).getParams()) {
+ stmt.accept(this);
+ String[] argCode = code.removeLast().split("=");
+ function.addAggregatorOptionParamId(argCode[0], counter++);
+ function.addAggregatorOptionParamInitializer(argCode[1]);
+ }
+ }
+ }
code.add("");
return;
}
@@ -1358,6 +1425,13 @@ public void visit(final VarDeclStatement n) {
n.getInitializer().accept(this);
String src = code.removeLast();
+ if(t instanceof BoaFunction) {
+ UserDefinedAgg.Builder functionDetails = UserDefinedAggregators.findByUserGivenName(n.getId().getToken());
+ if(functionDetails != null) {
+ functionDetails.lambdaInterface(src);
+ }
+ }
+
if(lhsType instanceof BoaTuple && t instanceof BoaArray) {
Operand op = n.getInitializer().getLhs().getLhs().getLhs().getLhs().getLhs().getOperand();
if(op instanceof Composite) {
@@ -1858,8 +1932,12 @@ public void visit(final OutputType n) {
final BoaTable t = (BoaTable) n.env.get(id);
if (n.getArgsSize() > 0) {
- n.getArg(0).accept(this);
- this.aggregators.put(id, new AggregatorDescription(aggregator, t.getType(), Arrays.asList(code.removeLast())));
+ List codegen = new ArrayList();
+ for(Expression e: n.getArgs()) {
+ e.accept(this);
+ codegen.add(code.removeLast());
+ }
+ this.aggregators.put(id, new AggregatorDescription(aggregator, t.getType(), codegen));
} else {
this.aggregators.put(id, new AggregatorDescription(aggregator, t.getType()));
}
diff --git a/src/java/boa/compiler/visitors/TypeCheckingVisitor.java b/src/java/boa/compiler/visitors/TypeCheckingVisitor.java
index c12c28126..f630dc882 100644
--- a/src/java/boa/compiler/visitors/TypeCheckingVisitor.java
+++ b/src/java/boa/compiler/visitors/TypeCheckingVisitor.java
@@ -391,9 +391,9 @@ else if (n.getExprsSize() > 0) {
if(!(checkTupleArray(types) == true)) {
final BoaType t = types.get(0);
- if (!(t instanceof BoaScalar))
- if (!(t instanceof BoaTuple))
- throw new TypeCheckException(n.getExprs(), "non-scalar/non-tuple type '" + t + "' can not be used in arrays");
+// if (!(t instanceof BoaScalar))
+// if (!(t instanceof BoaTuple))
+// throw new TypeCheckException(n.getExprs(), "non-scalar/non-tuple type '" + t + "' can not be used in arrays");
n.type = new BoaArray(t);
}
else
@@ -1314,9 +1314,9 @@ public void visit(final ArrayType n, final SymbolTable env) {
n.env = env;
n.getValue().accept(this, env);
final BoaType t = n.getValue().type;
- if (!(t instanceof BoaScalar))
- if (!(t instanceof BoaTuple))
- throw new TypeCheckException(n.getValue(), "non-scalar/non-tuple type '" + t + "' can not be used in arrays");
+// if (!(t instanceof BoaScalar))
+// if (!(t instanceof BoaTuple))
+// throw new TypeCheckException(n.getValue(), "non-scalar/non-tuple type '" + t + "' can not be used in arrays");
n.type = new BoaArray(t);
}
@@ -1400,6 +1400,10 @@ public void visit(final OutputType n, final SymbolTable env) {
if (n.getArgsSize() > 0 && annotation.formalParameters().length == 0)
throw new TypeCheckException(n.getArgs(), "output aggregator '" + n.getId().getToken() + "' takes no arguments");
+ for(VarDeclStatement stmt: n.getParams()) {
+ stmt.accept(this, env);
+ }
+
n.type = new BoaTable(type, indexTypes, tweight, annotation.canOmitWeight());
env.set(n.getId().getToken(), n.type);
n.getId().accept(this, env);
diff --git a/src/java/boa/datagen/forges/github/GetGithubRepoByUser.java b/src/java/boa/datagen/forges/github/GetGithubRepoByUser.java
index 6f4f30954..905386cee 100644
--- a/src/java/boa/datagen/forges/github/GetGithubRepoByUser.java
+++ b/src/java/boa/datagen/forges/github/GetGithubRepoByUser.java
@@ -33,7 +33,7 @@ public static void main(String[] args) {
// Arrays.sort(files, new Comparator() {
// @Override
// public int compare(File f1, File f2) {
-// int n1 = getNumber(f1.getName()), n2 = getNumber(f2.getName());
+// int n1 = getNumber(f1.getFuncVarName()), n2 = getNumber(f2.getFuncVarName());
// return n1 - n2;
// }
//
diff --git a/src/java/boa/functions/BoaAstIntrinsics.java b/src/java/boa/functions/BoaAstIntrinsics.java
index acd13cfe4..13026f4c2 100644
--- a/src/java/boa/functions/BoaAstIntrinsics.java
+++ b/src/java/boa/functions/BoaAstIntrinsics.java
@@ -469,9 +469,9 @@ protected boolean preVisit(Type node) {
// FIXME
/*
try {
- parseGenericType(BoaAstIntrinsics.type_name(node.getName()).trim(), map);
+ parseGenericType(BoaAstIntrinsics.type_name(node.getFuncVarName()).trim(), map);
} catch (final StackOverflowError e) {
- System.err.println("STACK ERR: " + node.getName() + " -> " + BoaAstIntrinsics.type_name(node.getName()).trim());
+ System.err.println("STACK ERR: " + node.getFuncVarName() + " -> " + BoaAstIntrinsics.type_name(node.getFuncVarName()).trim());
}
*/
return true;
diff --git a/src/java/boa/functions/BoaMLIntrinsics.java b/src/java/boa/functions/BoaMLIntrinsics.java
new file mode 100644
index 000000000..047df1fad
--- /dev/null
+++ b/src/java/boa/functions/BoaMLIntrinsics.java
@@ -0,0 +1,35 @@
+package boa.functions;
+
+import boa.BoaTup;
+import boa.datagen.util.FileIO;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+import javax.lang.model.element.Modifier;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.net.URLConnection;
+
+/**
+ * Machine learning utility function
+ * @author: nmtiwari
+ */
+public class BoaMLIntrinsics {
+ /**
+ * Return a random floating point number x in the range 0.0 < x < 1.0.
+ *
+ * @return A random floating point number x in the range 0.0 < x < 1.0
+ */
+ @FunctionSpec(name = "load", returnType = "tuple", formalParameters = { "string", "any"})
+ public static T load(String path, T type) {
+ Gson obj = new GsonBuilder().excludeFieldsWithModifiers(org.eclipse.jdt.core.dom.Modifier.TRANSIENT).serializeNulls().create();
+ String json = FileIO.readFileContents(new File(path));
+ T model = (T)obj.fromJson(json,type.getClass());
+ return model;
+ }
+}
diff --git a/src/java/boa/functions/BoaMatrixIntrinsics.java b/src/java/boa/functions/BoaMatrixIntrinsics.java
new file mode 100644
index 000000000..5e0275f75
--- /dev/null
+++ b/src/java/boa/functions/BoaMatrixIntrinsics.java
@@ -0,0 +1,1003 @@
+package boa.functions;
+
+import Jama.Matrix;
+import boa.BoaTup;
+
+import com.google.common.primitives.Doubles;
+import com.google.common.primitives.Longs;
+import org.apache.commons.lang.ArrayUtils;
+
+
+import java.lang.reflect.Array;
+import java.util.*;
+
+/**
+ * Created by nmtiwari on 2/11/17.
+ */
+public class BoaMatrixIntrinsics {
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "matrix", returnType = "array of array of int", formalParameters = { "array of int", "int" })
+ public static long[][] matrix(final long[] a, final long colsize) {
+ final int cols = (int)colsize;
+ final int rows = a.length/cols;
+ final long[][] result = new long[rows][cols];
+ for(int i = 0; i< rows; i++) {
+ for (int j = 0; j < cols; j++) {
+ result[i][j] = a[i * cols + j];
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "matrix", returnType = "array of array of string", formalParameters = { "array of string", "int" })
+ public static String[][] matrix(final String[] a, final long colsize) {
+ final int cols = (int)colsize;
+ final int rows = a.length/cols;
+ final String[][] result = new String[rows][cols];
+ for(int i = 0; i< rows; i++) {
+ for (int j = 0; j < cols; j++) {
+ result[i][j] = a[i * cols + j];
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "matrix", returnType = "array of array of float", formalParameters = { "array of float", "int" })
+ public static float[][] matrix(final float[] a, final long colsize) {
+ final int cols = (int)colsize;
+ final int rows = a.length/cols;
+ final float[][] result = new float[rows][cols];
+ for(int i = 0; i< rows; i++) {
+ for (int j = 0; j < cols; j++) {
+ result[i][j] = a[i * cols + j];
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "matrix", returnType = "array of array of tuple", formalParameters = { "array of tuple", "int" })
+ public static BoaTup[][] matrix(final BoaTup[] a, final long colsize) {
+ final int cols = (int)colsize;
+ final int rows = a.length/cols;
+ final BoaTup[][] result = new BoaTup[rows][cols];
+ for(int i = 0; i< rows; i++) {
+ for (int j = 0; j < cols; j++) {
+ result[i][j] = a[i * cols + j];
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of array of int", formalParameters = { "array of array of int", "array of array of int" })
+ public static long[][] multiply(final long[][] a, final long[][] b) {
+ double[][] a_double = new double[a.length][];
+ double[][] b_double = new double[b.length][];
+
+ for(int i = 0; i< a.length; i++){
+ a_double[i] = Doubles.toArray(Longs.asList(a[i]));
+ }
+
+ for(int i = 0; i< b.length; i++){
+ b_double[i] = Doubles.toArray(Longs.asList(b[i]));
+ }
+ Matrix a_matrix = new Matrix(a_double);
+ Matrix b_matrix = new Matrix(b_double);
+ double[][] result = a_matrix.times(b_matrix).getArray();
+ long[][]result_long = new long[result.length][];
+ for(int i = 0; i< result.length; i++){
+ result_long[i] = Longs.toArray(Doubles.asList(result[i]));
+ }
+ return result_long;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of array of int", formalParameters = { "array of array of int", "int" })
+ public static long[][] multiply(final long[][] a, final long b) {
+ double[][] a_double = new double[a.length][];
+
+ for(int i = 0; i< a.length; i++){
+ a_double[i] = Doubles.toArray(Longs.asList(a[i]));
+ }
+
+ Matrix a_matrix = new Matrix(a_double);
+ double[][] result = a_matrix.times(b).getArray();
+ long[][]result_long = new long[result.length][];
+ for(int i = 0; i< result.length; i++){
+ result_long[i] = Longs.toArray(Doubles.asList(result[i]));
+ }
+ return result_long;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of array of int", formalParameters = { "array of array of int", "float" })
+ public static double[][] multiply(final long[][] a, final float b) {
+ double[][] a_double = new double[a.length][];
+
+ for(int i = 0; i< a.length; i++){
+ a_double[i] = Doubles.toArray(Longs.asList(a[i]));
+ }
+
+ Matrix a_matrix = new Matrix(a_double);
+ return a_matrix.times(b).getArray();
+ }
+
+
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "matrixsum", returnType = "array of array of int", formalParameters = { "array of array of int", "array of array of int" })
+ public static long[][] matrixsum(final long[][] a, final long[][] b) {
+ double[][] a_double = new double[a.length][];
+ double[][] b_double = new double[b.length][];
+
+ for(int i = 0; i< a.length; i++){
+ a_double[i] = Doubles.toArray(Longs.asList(a[i]));
+ }
+
+ for(int i = 0; i< b.length; i++){
+ b_double[i] = Doubles.toArray(Longs.asList(b[i]));
+ }
+ Matrix a_matrix = new Matrix(a_double);
+ Matrix b_matrix = new Matrix(b_double);
+ double[][] result = a_matrix.plus(b_matrix).getArray();
+ long[][]result_long = new long[result.length][];
+ for(int i = 0; i< result.length; i++){
+ result_long[i] = Longs.toArray(Doubles.asList(result[i]));
+ }
+ return result_long;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "matrixsubstract", returnType = "array of array of int", formalParameters = { "array of array of int", "array of array of int" })
+ public static long[][] matrixsubstract(final long[][] a, final long[][] b) {
+ double[][] a_double = new double[a.length][];
+ double[][] b_double = new double[b.length][];
+
+ for(int i = 0; i< a.length; i++){
+ a_double[i] = Doubles.toArray(Longs.asList(a[i]));
+ }
+
+ for(int i = 0; i< b.length; i++){
+ b_double[i] = Doubles.toArray(Longs.asList(b[i]));
+ }
+ Matrix a_matrix = new Matrix(a_double);
+ Matrix b_matrix = new Matrix(b_double);
+ double[][] result = a_matrix.minus(b_matrix).getArray();
+ long[][]result_long = new long[result.length][];
+ for(int i = 0; i< result.length; i++){
+ result_long[i] = Longs.toArray(Doubles.asList(result[i]));
+ }
+ return result_long;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of array of float", formalParameters = { "array of array of float", "array of array of float" })
+ public static double[][] multiply(final double[][] a, final double[][] b) {
+ Matrix a_matrix = new Matrix(a);
+ Matrix b_matrix = new Matrix(b);
+ return a_matrix.times(b_matrix).getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of array of float", formalParameters = { "array of array of float", "float" })
+ public static double[][] multiply(final double[][] a, final double b) {
+ Matrix a_matrix = new Matrix(a);
+ return a_matrix.times(b).getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of array of float", formalParameters = { "array of array of float", "int" })
+ public static double[][] multiply(final double[][] a, final long b) {
+ Matrix a_matrix = new Matrix(a);
+ return a_matrix.times(b).getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of array of float", formalParameters = { "array of float", "array of array of float" })
+ public static double[][] multiply(final double[] a, final double[][] b) {
+ Matrix a_matrix = new Matrix(a, 1);
+ Matrix b_matrix = new Matrix(b);
+ return a_matrix.times(b_matrix).getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of float", formalParameters = { "array of float", "float" })
+ public static double[] multiply(final double[] a, final double b) {
+ Matrix a_matrix = new Matrix(a, a.length);
+ return a_matrix.times(b).getArray()[0];
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of float", formalParameters = { "array of float", "int" })
+ public static double[] multiply(final double[] a, final long b) {
+ Matrix a_matrix = new Matrix(a, 1);
+ double[][] temp = a_matrix.times(b).getArray();;
+ return temp[0];
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "multiply", returnType = "array of array of float", formalParameters = { "array of array of float", "array of float" })
+ public static double[][] multiply(final double[][] a, final double[] b) {
+ Matrix a_matrix = new Matrix(a);
+ Matrix b_matrix = new Matrix(b, 1);
+ return a_matrix.times(b_matrix).getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "matrixsum", returnType = "array of array of float", formalParameters = { "array of array of float", "array of array of float" })
+ public static double[][] matrixsum(final double[][] a, final double[][] b) {
+ Matrix a_matrix = new Matrix(a);
+ Matrix b_matrix = new Matrix(b);
+ return a_matrix.plus(b_matrix).getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "matrixsum", returnType = "array of float", formalParameters = { "array of float", "array of float" })
+ public static double[] matrixsum(final double[] a, final double[] b) {
+ if(a.length != b.length) {
+ throw new IllegalArgumentException("Argument lengths are not equal");
+ }
+ double[] result = new double[a.length];
+ for(int i =0 ; i < a.length; i++) {
+ result[i] = a[i] + b[i];
+ }
+ return result;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "matrixsubstract", returnType = "array of array of float", formalParameters = { "array of array of float", "array of array of float" })
+ public static double[][] matrixsubstract(final double[][] a, final double[][] b) {
+ Matrix a_matrix = new Matrix(a);
+ Matrix b_matrix = new Matrix(b);
+ return a_matrix.minus(b_matrix).getArray();
+ }
+
+
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "flatten", returnType = "array of float", formalParameters = { "array of array of float" })
+ public static double[] flatten(final double[][] a) {
+ final double[] flattenedTuples = new double[a.length * a[0].length];
+ int counter = 0;
+ for(int i = 0; i< a.length; i++) {
+ for (int j = 0; j < a[i].length; j++) {
+ flattenedTuples[counter++] = a[i][j];
+ }
+ }
+ return flattenedTuples;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "flatten", returnType = "array of tuple", formalParameters = { "array of array of tuple" })
+ public static BoaTup[] flatten(final BoaTup[][] a) {
+ final BoaTup[] flattenedTuples = new BoaTup[a.length * a[0].length];
+ int counter = 0;
+ for(int i = 0; i< a.length; i++) {
+ for (int j = 0; j < a[i].length; j++) {
+ flattenedTuples[counter++] = a[i][j];
+ }
+ }
+ return flattenedTuples;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "flatten", returnType = "array of float", formalParameters = { "tuple" })
+ public static double[] flatten(final BoaTup a) {
+ final int size = tupleLength(a);
+ final double[] results = new double[size];
+ int i = 0;
+ for(Double ele: a.asArray(new Double[1])){
+ results[i] = ele;
+ }
+ return results;
+ }
+
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "flattenedMatrix", returnType = "array of array of float", formalParameters = { "array of tuple", "int" })
+ public static double[][] flattenedMatrix(final BoaTup[] a, final long colsize) {
+ final List flattenedTuples = new ArrayList();
+ for(int i = 0; i< a.length; i++) {
+ for(Double ele: a[i].asArray(new Double[1])){
+ flattenedTuples.add(ele);
+ }
+ }
+
+ final int cols = (int)colsize;
+ final int rows = flattenedTuples.size()/cols;
+ final double[][] result = new double[rows][cols];
+
+ for(int i = 0; i< rows; i++) {
+ for (int j = 0; j < cols; j++) {
+ result[i][j] = flattenedTuples.get(i * cols + j);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "tupleLength", returnType = "int", formalParameters = { "tuple" })
+ public static int tupleLength(final BoaTup a) {
+ final int sizeOfTuple = a.asArray(new Double[1]).length;
+ return sizeOfTuple;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "means", returnType = "array of float", formalParameters = { "array of tuple"})
+ public static double[] means(final BoaTup[] a) {
+ final int sizeOfTuple = a[0].asArray(new Double[1]).length;
+ final double[] means = new double[sizeOfTuple];
+
+ for(int i = 0; i< a.length; i++) {
+ Double[] tupAsArr = a[i].asArray(new Double[1]);
+ for (int index = 0; index < sizeOfTuple; index++) {
+ means[index] = means[index] + tupAsArr[index];
+ }
+ }
+
+ for (int index = 0; index < sizeOfTuple; index++) {
+ means[index] = means[index] / a.length;
+ }
+ return means;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "sum", returnType = "array of float", formalParameters = { "array of tuple"})
+ public static double[] sum(final BoaTup[] a) {
+ final int sizeOfTuple = a[0].asArray(new Double[1]).length;
+ final double[] sum = new double[sizeOfTuple];
+
+ for(int i = 0; i< a.length; i++) {
+ Double[] tupAsArr = a[i].asArray(new Double[1]);
+ for (int index = 0; index < sizeOfTuple; index++) {
+ sum[index] = sum[index] + tupAsArr[index];
+ }
+ }
+
+ return sum;
+ }
+
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "sum", returnType = "array of float", formalParameters = { "tuple", "tuple"})
+ public static double[] sum(final BoaTup a, final BoaTup b) {
+ final int sizeOfTupleA = a.asArray(new Double[1]).length;
+ final int sizeOfTupleB = b.asArray(new Double[1]).length;
+ if(sizeOfTupleA != sizeOfTupleB) {
+ throw new IllegalArgumentException("Dissimilar sttributes in Tuples");
+ }
+ final double[] sum = new double[sizeOfTupleA];
+ for(int i = 0; i< sizeOfTupleA; i++) {
+ Double[] tupAsArrA = a.asArray(new Double[1]);
+ Double[] tupAsArrB = a.asArray(new Double[1]);
+ for (int index = 0; index < sizeOfTupleA; index++) {
+ sum[index] = tupAsArrB[index] + tupAsArrA[index];
+ }
+ }
+ return sum;
+ }
+
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "flatten", returnType = "array of int", formalParameters = { "array of array of int" })
+ public static long[] flatten(final long[][] a) {
+ final long[] flattenedTuples = new long[a.length * a[0].length];
+ int counter = 0;
+ for(int i = 0; i< a.length; i++) {
+ for (int j = 0; j < a[i].length; j++) {
+ flattenedTuples[counter++] = a[i][j];
+ }
+ }
+ return flattenedTuples;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "transpose", returnType = "array of array of float", formalParameters = { "array of array of float" })
+ public static double[][] transpose(final double[][] a) {
+ Matrix matrix = new Matrix(a);
+ matrix = matrix.transpose();
+ return matrix.getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "transpose", returnType = "array of array of float", formalParameters = { "array of float" })
+ public static double[][] transpose(final double[] a) {
+ double[][] data = new double[a.length][];
+ for(int i = 0; i < a.length; i++){
+ double[] __temp = {a[i]};
+ data[i] = __temp;
+ }
+ return data;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "eigenvalsReal", returnType = "array of float", formalParameters = { "array of array of float" })
+ public static double[] eigenvalsReal(final double[][] a) {
+ Matrix matrix = new Matrix(a);
+ double[] temp = matrix.eig().getRealEigenvalues();
+ return matrix.eig().getRealEigenvalues();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "eigenvalsImg", returnType = "array of float", formalParameters = { "array of array of float" })
+ public static double[] eigenvalsImg(final double[][] a) {
+ Matrix matrix = new Matrix(a);
+ return matrix.eig().getImagEigenvalues();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "eigenvectors", returnType = "array of array of float", formalParameters = { "array of array of float" })
+ public static double[][] eigenvectors(final double[][] a) {
+ Matrix matrix = new Matrix(a);
+ return matrix.eig().getV().getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "vector", returnType = "array of array of float", formalParameters = { "array of float" })
+ public static double[][] vector(final double[] a) {
+ double[][] data = new double[a.length][];
+ for(int i = 0; i < a.length; i++){
+ double[] __temp = {a[i]};
+ data[i] = __temp;
+ }
+ return data;
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "inverse", returnType = "array of array of float", formalParameters = { "array of array of float" })
+ public static double[][] inverse(final double[][] a) {
+ Matrix matrix = new Matrix(a);
+ matrix = matrix.inverse();
+ return matrix.getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "inverse", returnType = "array of array of float", formalParameters = { "array of array of int" })
+ public static double[][] inverse(final long[][] a) {
+ double[][] data = new double[a.length][];
+ for(int i = 0; i< data.length; i++){
+ data[i] = Doubles.toArray(Longs.asList(a[i]));
+ }
+ Matrix matrix = new Matrix(data);
+ matrix = matrix.inverse();
+ return matrix.getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array.
+ *
+ * @param a
+ * An array of array of int
+ *
+ * @return A sub matrix from Matrix a
+ */
+ @FunctionSpec(name = "submatrix", returnType = "array of array of int", formalParameters = { "array of array of int", "int", "int", "int", "int" })
+ public static long[][] submatrix(final long[][] a, final long r_start, final long r_end, final long c_start, final long c_end) {
+ double[][] data = new double[a.length][];
+ for(int i = 0; i< data.length; i++){
+ data[i] = Doubles.toArray(Longs.asList(a[i]));
+ }
+ Matrix matrix = new Matrix(data);
+ matrix = matrix.getMatrix((int)r_start, (int)r_end, (int)c_start, (int)c_end);
+ data = matrix.getArray();
+ long[][] result = new long[a.length][];
+ for(int i = 0; i< data.length; i++){
+ result[i] = Longs.toArray(Doubles.asList(data[i]));
+ }
+ return result;
+ }
+
+ /**
+ * Returns the matrix version of an array.
+ *
+ * @param a
+ * An array of array of int
+ *
+ * @return A sub matrix from Matrix a
+ */
+ @FunctionSpec(name = "submatrix", returnType = "array of array of float", formalParameters = { "array of array of float", "int", "int", "int", "int" })
+ public static double[][] submatrix(final double[][] a, final long r_start, final long r_end, final long c_start, final long c_end) {
+ Matrix matrix = new Matrix(a);
+ matrix = matrix.getMatrix((int)r_start, (int)r_end, (int)c_start, (int)c_end);
+ return matrix.getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array.
+ *
+ * @param a
+ * An array of array of int
+ *
+ * @return A sub matrix from Matrix a
+ */
+ @FunctionSpec(name = "identity", returnType = "array of array of float", formalParameters = { "int", "int" })
+ public static double[][] identity(final int row, final int col) {
+ Matrix matrix = Matrix.identity(row, col);
+ return matrix.getArray();
+ }
+
+ /**
+ * Returns the matrix version of an array.
+ *
+ * @param a
+ * An array of array of int
+ *
+ * @return A sub matrix from Matrix a
+ */
+ @FunctionSpec(name = "getCol", returnType = "array of float", formalParameters = { "array of array of float", "int" })
+ public static double[] getCol(final double[][] data, final long col) {
+ double[] result = new double[data.length];
+ for(int i = 0; i < result.length; i++ ) {
+ result[i] = data[i][(int)col];
+ }
+ return result;
+ }
+
+ /**
+ * Returns the matrix version of an array.
+ *
+ * @param a
+ * An array of array of int
+ *
+ * @return A sub matrix from Matrix a
+ */
+ @FunctionSpec(name = "unique", returnType = "array of float", formalParameters = { "array of float" })
+ public static double[] unique(final double[] data) {
+ Set set = new HashSet(Doubles.asList(data));
+ final double[] result = new double[set.size()];
+ int i = 0;
+ for(Double d: set) {
+ result[i++] = d;
+ }
+ return result;
+ }
+
+
+ /**
+ * Returns the matrix version of an array.
+ *
+ * @param a
+ * An array of array of int
+ *
+ * @return A sub matrix from Matrix a
+ */
+ @FunctionSpec(name = "unique", returnType = "array of int", formalParameters = { "array of int" })
+ public static long[] unique(final long[] data) {
+ Set set = new HashSet(Longs.asList(data));
+ final Long[] result = set.toArray(new Long[data.length]);
+ return ArrayUtils.toPrimitive(result);
+ }
+
+ /**
+ * Returns the matrix version of an array.
+ *
+ * @param a
+ * An array of array of int
+ *
+ * @return A sub matrix from Matrix a
+ */
+ @FunctionSpec(name = "getRow", returnType = "array of float", formalParameters = { "array of array of float", "int" })
+ public static double[] getRow(final double[][] data, final long row) {
+ return data[(int)row];
+ }
+
+ /**
+ * Returns the matrix version of an array.
+ *
+ * @param a
+ * An array of array of int
+ *
+ * @return A sub matrix from Matrix a
+ */
+ @FunctionSpec(name = "getCol", returnType = "array of int", formalParameters = { "array of array of int", "int" })
+ public static long[] getCol(final long[][] data, final long col) {
+ long[] result = new long[data[0].length];
+ for(int i = 0; i < data[0].length; i++ ) {
+ result[i] = data[i][(int)col];
+ }
+ return result;
+ }
+
+
+ /**
+ * Returns the matrix version of an array.
+ *
+ * @param a
+ * An array of array of int
+ *
+ * @return A sub matrix from Matrix a
+ */
+ @FunctionSpec(name = "getRow", returnType = "array of int", formalParameters = { "array of array of int", "int" })
+ public static long[] getRow(final long[][] data, final int row) {
+ return data[row];
+ }
+
+
+ /**
+ * Returns the matrix version of an array. Only scalar values can be sorted.
+ * Values will be arranged in increasing order. (An optional comparison
+ * function, which takes two elements and returns int {-,0,+}, is accepted
+ * as a second argument, but it is curently ignored.)
+ *
+ * @param a
+ * An array of long
+ *
+ * @return A sorted copy of a
+ */
+ @FunctionSpec(name = "meanScaling", returnType = "array of array of float", formalParameters = { "array of array of float", "array of float" })
+ public static double[][] meanScaling(final double[][] a, final double[] b) {
+ for(int i = 0; i < a.length; i++) {
+ for(int j = 0; j < a[i].length; j++) {
+ a[i][j] = a[i][j] - b[j];
+ }
+ }
+ Matrix a_matrix = new Matrix(a);
+ return a_matrix.getArray();
+ }
+
+
+
+}
diff --git a/src/java/boa/functions/BoaSpecialIntrinsics.java b/src/java/boa/functions/BoaSpecialIntrinsics.java
index 87f92f541..2ecf01933 100644
--- a/src/java/boa/functions/BoaSpecialIntrinsics.java
+++ b/src/java/boa/functions/BoaSpecialIntrinsics.java
@@ -16,6 +16,8 @@
*/
package boa.functions;
+import boa.BoaTup;
+
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
@@ -141,6 +143,45 @@ public static String[] newBoolean(String[] a, long size, String val) {
return arr;
}
+ @FunctionSpec(name = "new", returnType = "array of array of float", formalParameters = { "array of array of float", "int", "array of float" })
+ public static double[][] newTuple(double[][] a, long size, double[] val) {
+ double[][] arr = new double[(int)size][];
+ for (int i = 0; i < size; i++)
+ arr[i] = val;
+ return arr;
+ }
+
+ @FunctionSpec(name = "new", returnType = "array of tuple", formalParameters = { "array of tuple", "int" })
+ public static T[] newTuple(T[] a, long size) {
+ Object[] arr = new Object[(int)size];
+ for (int i = 0; i < size; i++)
+ arr[i] = null;
+ return (T[]) arr;
+ }
+
+ @FunctionSpec(name = "new", returnType = "array of array of array of float", formalParameters = { "array of array of array of float", "int", "array of array of float" })
+ public static double[][][] newTuple(double[][][] a, long size, double[][] val) {
+ double[][][] arr = new double[(int)size][][];
+ for (int i = 0; i < size; i++)
+ arr[i] = val;
+ return arr;
+ }
+
+
+ @FunctionSpec(name = "print", returnType = "bool", formalParameters = { "any"})
+ public static boolean print(HashMap a) {
+ for (Object k: a.keySet()) {
+ System.out.println(k);
+ }
+ return true;
+ }
+
+ @FunctionSpec(name = "printany", returnType = "int", formalParameters = { "any"})
+ public static boolean print(Object a) {
+ System.out.println("nitin: " + a);
+ return true;
+ }
+
public static String regex(final String type, final long base) {
if (BoaSpecialIntrinsics.regexMap.containsKey(type + "," + base))
return BoaSpecialIntrinsics.regexMap.get(type + "," + base);
@@ -154,4 +195,18 @@ public static String regex(final String type) {
else
throw new RuntimeException("unimplemented");
}
+
+ @FunctionSpec(name = "convertFloatArrayToInt", returnType = "array of int", formalParameters = { "array of float" })
+ public static long[] convertFloatArrayToInt(final double[] inp) {
+ long[] converted = new long[inp.length];
+ for(int i = 0; i < inp.length; i++) {
+ converted[i] = Math.round(inp[i]);
+ }
+ return converted;
+ }
+
+ @FunctionSpec(name = "roundToLong", returnType = "int", formalParameters = { "float" })
+ public static long roundToLong(final double inp) {
+ return Math.round(inp);
+ }
}
diff --git a/src/java/boa/io/EmitValue.java b/src/java/boa/io/EmitValue.java
index 8f324ded4..148dce6f6 100644
--- a/src/java/boa/io/EmitValue.java
+++ b/src/java/boa/io/EmitValue.java
@@ -19,12 +19,15 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.io.ByteArrayInputStream;
+import java.io.ObjectInputStream;
import java.util.Arrays;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import boa.functions.BoaCasts;
+import boa.BoaTup;
/**
* A {@link Writable} that contains a datum and an optional metadatum to be
@@ -36,6 +39,8 @@
public class EmitValue implements Writable {
private String[] data;
private String metadata;
+ private String[] odata;
+ private BoaTup tdata;
/**
* Construct an EmitValue.
@@ -233,6 +238,85 @@ public EmitValue(final double data, final double metadata) {
this(new String[] { BoaCasts.doubleToString(data) }, BoaCasts.doubleToString(metadata));
}
+ /**
+ * Construct an EmitValue.
+ *
+ * @param data
+ * A {@link BoaTup} containing the data to be emitted
+ * @param metadata
+ * A {@link String} containing the metadata to be emitted
+ */
+ public EmitValue(final BoaTup data, final String metadata) {
+ this.tdata = data;
+ }
+
+ /**
+ * Construct an EmitValue.
+ *
+ * @param data
+ * A {@link BoaTup} containing the data to be emitted
+ */
+ public EmitValue(final BoaTup data) {
+ this(data, null);
+ }
+
+
+ /**
+ * Construct an EmitValue.
+ *
+ * @param data
+ * An array of {@link double} containing the data to be emitted
+ * @param metadata
+ * A {@link String} containing the metadata to be emitted
+ */
+ public EmitValue(final double[] data, final String metadata) {
+ final String[] strings = new String[data.length];
+
+ for (int i = 0; i < data.length; i++)
+ strings[i] = String.valueOf(data[i]);
+
+ this.data = strings;
+ this.metadata = metadata;
+ }
+
+ /**
+ * Construct an EmitValue.
+ *
+ * @param data
+ * An array of {@link double} containing the data to be emitted
+ */
+ public EmitValue(final double[] data) {
+ this(data, null);
+ }
+
+ /**
+ * Construct an EmitValue.
+ *
+ * @param data
+ * An array of {@link long} containing the data to be emitted
+ * @param metadata
+ * A {@link String} containing the metadata to be emitted
+ */
+ public EmitValue(final long[] data, final String metadata) {
+ final String[] strings = new String[data.length];
+
+ for (int i = 0; i < data.length; i++)
+ strings[i] = String.valueOf(data[i]);
+
+ this.data = strings;
+ this.metadata = metadata;
+ }
+
+ /**
+ * Construct an EmitValue.
+ *
+ * @param data
+ * An array of {@link long} containing the data to be emitted
+ */
+ public EmitValue(final long[] data) {
+ this(data, null);
+ }
+
/** {@inheritDoc} */
@Override
public void readFields(final DataInput in) throws IOException {
@@ -247,20 +331,46 @@ public void readFields(final DataInput in) throws IOException {
this.metadata = null;
else
this.metadata = metadata;
+
+ final int length = in.readInt();
+ if(length > 0) {
+ byte[] bytes = new byte[length];
+ in.readFully(bytes, 0, length);
+ ByteArrayInputStream bin = new ByteArrayInputStream(bytes);
+ ObjectInputStream dataIn = new ObjectInputStream(bin);
+ Object o = null;
+ try {
+ o = dataIn.readObject();
+ } catch(Exception e) {
+ e.printStackTrace();
+ }
+ this.tdata = (BoaTup)o;
+ }
}
/** {@inheritDoc} */
@Override
public void write(final DataOutput out) throws IOException {
- out.writeInt(this.data.length);
-
- for (final String d : this.data)
- Text.writeString(out, d);
+ if(this.data == null)
+ out.writeInt(0);
+ else {
+ out.writeInt(this.data.length);
+ for (final String d : this.data)
+ Text.writeString(out, d);
+ }
if (this.metadata == null)
Text.writeString(out, "");
else
Text.writeString(out, this.metadata);
+
+ if (this.tdata == null)
+ out.writeInt(0);
+ else {
+ byte[] serializedObject = this.tdata.serialize(this.tdata);
+ out.writeInt(serializedObject.length);
+ out.write(serializedObject);
+ }
}
/**
@@ -285,6 +395,10 @@ public String getMetadata() {
return this.metadata;
}
+ public BoaTup getTuple() {
+ return this.tdata;
+ }
+
/**
* @param metadata
* the metadatum to set
diff --git a/src/java/boa/runtime/BoaCombiner.java b/src/java/boa/runtime/BoaCombiner.java
index 81d131e10..e7d6c30ea 100644
--- a/src/java/boa/runtime/BoaCombiner.java
+++ b/src/java/boa/runtime/BoaCombiner.java
@@ -95,8 +95,12 @@ protected void reduce(final EmitKey key, final Iterable values, final
for (final EmitValue value : values)
try {
- for (final String s : value.getData())
- a.aggregate(s, value.getMetadata());
+ if(value.getTuple() != null) {
+ a.aggregate(value.getTuple(), value.getMetadata());
+ } else {
+ for (final String s : value.getData())
+ a.aggregate(s, value.getMetadata());
+ }
} catch (final FinishedException e) {
// we are done
return;
diff --git a/src/java/boa/runtime/BoaReducer.java b/src/java/boa/runtime/BoaReducer.java
index 103249343..efa192cad 100644
--- a/src/java/boa/runtime/BoaReducer.java
+++ b/src/java/boa/runtime/BoaReducer.java
@@ -87,8 +87,13 @@ protected void reduce(final EmitKey key, final Iterable values, final
for (final EmitValue value : values)
try {
- for (final String s : value.getData())
- a.aggregate(s, value.getMetadata());
+ if(value.getTuple() != null) {
+ a.aggregate(value.getTuple(), value.getMetadata());
+ }else{
+ for (final String s : value.getData())
+ a.aggregate(s, value.getMetadata());
+ }
+
} catch (final FinishedException e) {
// we are done
return;
diff --git a/src/test/boa/test/compiler/TestTraversalBad.java b/src/test/boa/test/compiler/TestTraversalBad.java
index 1696db43a..36c62b31e 100644
--- a/src/test/boa/test/compiler/TestTraversalBad.java
+++ b/src/test/boa/test/compiler/TestTraversalBad.java
@@ -37,6 +37,6 @@ public void traversalWithStop() throws IOException {
@Test
public void traversalWithNoReturn() throws IOException {
- codegen(load(badDir + "traverse-with-no-return-statement.boa"), "Error on line 134: missing return statement");
+ codegen(load(badDir + "traverse-with-no-return-statement.boa"), "Error on line 135: missing return statement");
}
}
diff --git a/src/test/boa/test/compiler/TestTypecheckBad.java b/src/test/boa/test/compiler/TestTypecheckBad.java
index ae81e894d..adfa9acfe 100644
--- a/src/test/boa/test/compiler/TestTypecheckBad.java
+++ b/src/test/boa/test/compiler/TestTypecheckBad.java
@@ -88,16 +88,16 @@ public void currentBadType() throws IOException {
@Test
public void complexArray1() throws IOException {
- typecheck(load(badDir + "complex-arrays.boa"), "non-scalar/non-tuple type 'set of int' can not be used in arrays");
+ typecheck(load(badDir + "complex-arrays.boa"), "No emit statements detected - there will be no output generated");
}
@Test
public void complexArray2() throws IOException {
- typecheck(load(badDir + "complex-arrays2.boa"), "non-scalar/non-tuple type 'stack of int' can not be used in arrays");
+ typecheck(load(badDir + "complex-arrays2.boa"), "No emit statements detected - there will be no output generated");
}
@Test
public void complexArray3() throws IOException {
- typecheck(load(badDir + "complex-arrays3.boa"), "non-scalar/non-tuple type 'map[string] of int' can not be used in arrays");
+ typecheck(load(badDir + "complex-arrays3.boa"), "No emit statements detected - there will be no output generated");
}
}
diff --git a/templates/BoaJava.stg b/templates/BoaJava.stg
index 3dd98c3e6..2834831db 100644
--- a/templates/BoaJava.stg
+++ b/templates/BoaJava.stg
@@ -124,25 +124,83 @@ new ()
>>
TupleType(name, fields, types) ::= <<
-private class
+ static class implements boa.BoaTup, java.io.Serializable
{
___}; separator="; ">;
( ___}; separator=", "> ){
- = new (___)}; separator="; ">;
+ = (___)}; separator="; ">;
}
( tmp){
- = new (tmp.___)}; separator="; ">;
+ = (tmp.___)}; separator="; ">;
}
public clone() {
return new (this);
}
+
+ public byte[] serialize(Object o) throws java.io.IOException {
+ java.io.ByteArrayOutputStream byteOutStream = new java.io.ByteArrayOutputStream();
+ java.io.ObjectOutputStream objectOut = new java.io.ObjectOutputStream(byteOutStream);
+ objectOut.writeObject(o);
+ objectOut.close();
+ return byteOutStream.toByteArray();
+ }
+
+ public static deSerialize(String serializedObject) throws java.io.IOException {
+ obj = null;
+ try {
+ byte b[] = serializedObject.getBytes();
+ java.io.ByteArrayInputStream bi = new java.io.ByteArrayInputStream(b);
+ java.io.ObjectInputStream si = new java.io.ObjectInputStream(bi);
+ obj = () si.readObject();
+ } catch (Exception e) {
+ System.out.println(e);
+ }
+ return obj;
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer result = new StringBuffer();
+ result.append("{");
+ for(String str: getValues()) {
+ result.append(str).append(", ");
+ }
+ result.deleteCharAt(result.length() - 1);
+ result.deleteCharAt(result.length() - 1);
+ result.append("}");
+ return result.toString();
+ }
+
+ public String[] getValues() {
+ return new String[] {)}; separator=", "> };
+ }
+
+ public String[] getFieldNames() {
+ return new String[] {"}; separator=", "> };
+ }
+
+ public Object getValue(String f) {
+ "))
+ return ___;
+ }; separator=" ">
+ return "empty";
+ }
+
+ @Override
+ public \ T[] asArray(T[] type) {
+ java.util.List\ _temp = new java.util.ArrayList\();
+ );}; separator="\n">
+ return _temp.toArray(type);
+ }
+
}<\n>
>>
Tuple(name, exprlist) ::= "new () <\n>"
EnumType(ename, fields, values, fname) ::= <<
-private enum
+private enum implements boa.BoaEnumInterface
{
()}; separator=", ">;
diff --git a/templates/BoaJavaHadoop.stg b/templates/BoaJavaHadoop.stg
index 156e8fa20..ea983f342 100644
--- a/templates/BoaJavaHadoop.stg
+++ b/templates/BoaJavaHadoop.stg
@@ -1,6 +1,8 @@
-Program(name, numreducers, jobs, jobnames, combineTables, reduceTables, splitsize, isLocal) ::= <<
+Program(name, numreducers, jobs, jobnames, combineTables, reduceTables, splitsize, isLocal, userDeclAgg) ::= <<
package boa;
+<\n>}>
+
public class extends boa.runtime.BoaRunner {
/** {@inheritDoc} */
@Override
@@ -171,14 +173,14 @@ public class extends boa.runtime.BoaRunner {
static class BoaCombiner extends boa.runtime.BoaCombiner {
public BoaCombiner() {
super();
-
+ <\n>}>
<\n>}> }
}
static class BoaReducer extends boa.runtime.BoaReducer {
public BoaReducer() {
super();
-
+ <\n>}>
<\n>}> }
}
@@ -200,7 +202,7 @@ public class extends boa.runtime.BoaRunner {
>>
Job(name, staticDeclarations, staticStatements, statements) ::= <<
-private static class Job implements BoaJob {
+ static class Job implements BoaJob {
{
diff --git a/templates/UserAggregatorClass.stg b/templates/UserAggregatorClass.stg
new file mode 100644
index 000000000..228d2eef4
--- /dev/null
+++ b/templates/UserAggregatorClass.stg
@@ -0,0 +1,74 @@
+AggregatorClass(funcName, funcArg, lambdaType, lambdaName, interface, lambdaInit, methods, aggParams, constcode) ::= <<
+class UserDefined$ extends boa.aggregators.UserDefinedAggregator {
+ java.util.List\<\> _$values;
+ <";">}>
+ ;
+
+
+
+ private
+
+ public
+
+ UserDefined$() {
+ = ;
+ }
+
+
+ public UserDefined$
+
+
+}
+>>
+
+OverridenMethods(name, type, nonPrimitive, parser, javaPrimitiveType, returnType) ::= <<
+ @Override
+ public void aggregate(String data, String metadata) throws java.io.IOException,java.lang.InterruptedException, boa.aggregators.FinishedException {
+
+ this._$values.add(.deSerialize(data));
+
+ this._$values.add((data));
+
+ }
+
+
+
+ @Override
+ public void aggregate( data, String metadata) throws java.io.IOException,java.lang.InterruptedException, boa.aggregators.FinishedException {
+ this._$values.add((data));
+ }
+
+ @Override
+ public void aggregate( data) throws java.io.IOException,java.lang.InterruptedException, boa.aggregators.FinishedException {
+ aggregate(data);
+ }
+
+
+ @Override
+ public void finish() throws java.io.IOException, java.lang.InterruptedException {
+ try{
+
+ result = .invoke(this._$values.toArray(new [this._$values.size()]));
+
+ result = .invoke(org.apache.commons.lang.ArrayUtils.toPrimitive(this._$values.toArray(new [this._$values.size()])));
+
+ this.collect(result);
+ super.store(result);
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void start(final boa.io.EmitKey key) {
+ _$values = new java.util.ArrayList\<\>();
+ super.start(key);
+ }
+
+
+ @Override
+ public void aggregate(BoaTup data, String metadata) throws java.io.IOException, java.lang.InterruptedException, boa.aggregators.FinishedException {
+ this._$values.add(()data);
+ }
+
+>>
\ No newline at end of file
diff --git a/test/datagen/datagen.iml b/test/datagen/datagen.iml
new file mode 100644
index 000000000..60c83f1fd
--- /dev/null
+++ b/test/datagen/datagen.iml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/test/lda.boa b/test/lda.boa
new file mode 100644
index 000000000..270f92e3f
--- /dev/null
+++ b/test/lda.boa
@@ -0,0 +1,152 @@
+# Counting the 10 most used programming languages
+p: Project = input;
+counts: output top(10) of string weight int;
+astCount: int = 0;
+type fv = {a:float, b:float, c:float, d:float};
+type sol = {eigenvals: array of float, eigenvectors: array of array of float};
+
+
+lda:= function(vals: array of fv): array of float {
+ dataset : array of array of float = submatrix(flattenedMatrix(vals, 4), 0, len(vals) -1, 0, 2);
+ labels: array of float = getCol(flattenedMatrix(vals, 4), 3);
+ uniqueLables: array of int = convertFloatArrayToInt(unique(labels));
+
+
+ dummy: array of array of float = {{0.0, 0.0, 0.0}};
+ classified: array of array of array of float; #
+ classified = new(classified, len(uniqueLables), dummy);
+ foreach(i:int; abs(uniqueLables[i])) {
+ if(def(classified) && def(classified[roundToLong(labels[i])])) {
+ classified[roundToLong(labels[i])] = classified[roundToLong(labels[i])] + {dataset[i]};
+ } else if(def(classified)) {
+ classified[roundToLong(labels[i])] = {dataset[i]};
+ }
+ }
+
+
+ # calculate group mean
+ groupmeans : array of array of float;
+ groupmeans = new(groupmeans, len(uniqueLables), {0.0, 0.0, 0.0});
+ foreach(i: int; groupmeans[i]) {
+ foreach(j:int; def(classified[i][j])) {
+ groupmeans[i][0] = classified[i][j][0];
+ groupmeans[i][1] = classified[i][j][1];
+ groupmeans[i][2] = classified[i][j][2];
+ }
+ size : int = len(classified[i]);;
+ groupmeans[i][0] = groupmeans[i][0] / size;
+ groupmeans[i][1] = groupmeans[i][1] / size;
+ groupmeans[i][2] = groupmeans[i][2] / size;
+ }
+
+ # calculate global means
+ globalMean : array of float;
+ globalMean = new(globalMean, 3, 0.0);
+ foreach(j : int; dataset[j]) {
+ globalMean[0] = globalMean[0] + dataset[j][0];
+ globalMean[1] = globalMean[1] + dataset[j][1];
+ globalMean[2] = globalMean[2] + dataset[j][2];
+ }
+
+ globalMean[0] = globalMean[0] / len(dataset);
+ globalMean[1] = globalMean[1] / len(dataset);
+ globalMean[2] = globalMean[2] / len(dataset);
+
+
+ # correct subset data
+ foreach(i:int; def(classified[i])) {
+ foreach(j:int; classified[i][j]) {
+ classified[i][j][0] = classified[i][j][0] - globalMean[0];
+ classified[i][j][1] = classified[i][j][1] - globalMean[1];
+ classified[i][j][2] = classified[i][j][2] - globalMean[2];
+ }
+ }
+
+ # calculate covariance
+ covariance : array of array of array of float;
+ covariance = new(covariance, len(uniqueLables), {{0.0,0.0,0.0}, {0.0,0.0,0.0}, {0.0,0.0,0.0}});
+ foreach(i:int; uniqueLables[i]) {
+ foreach(j:int; covariance[i][j]) {
+ foreach(l:int; classified[l]) {
+ covariance[i][j][0] = covariance[i][j][0] + classified[i][l][j] * classified[i][l][0];
+ covariance[i][j][1] = covariance[i][j][1] + classified[i][l][j] * classified[i][l][1];
+ covariance[i][j][2] = covariance[i][j][2] + classified[i][l][j] * classified[i][l][2];
+ }
+ covariance[i][j][0] = covariance[i][j][0] / len(classified[i]);
+ covariance[i][j][1] = covariance[i][j][1] / len(classified[i]);
+ covariance[i][j][2] = covariance[i][j][2] / len(classified[i]);
+ }
+ }
+
+ # calculate pooled within group covariance matrix and invert it
+ pooledInverseCovariance : array of array of float;
+ pooledInverseCovariance = new(pooledInverseCovariance, len(uniqueLables), {0.0,0.0,0.0});
+ totalData: int = len(dataset);
+ foreach(j:int; pooledInverseCovariance[j]){
+ foreach(l:int; uniqueLables[l]) {
+ pooledInverseCovariance[j][0] = pooledInverseCovariance[j][0] + len(classified[l]) / totalData * covariance[l][j][0];
+ pooledInverseCovariance[j][1] = pooledInverseCovariance[j][1] + len(classified[l]) / totalData * covariance[l][j][1];
+ pooledInverseCovariance[j][2] = pooledInverseCovariance[j][2] + len(classified[l]) / totalData * covariance[l][j][2];
+ }
+ }
+
+ #TODO inverse the pooledInverseCovariance
+
+ probability: array of float;
+ probability = new(probability, len(uniqueLables), 0.0);
+
+ return probability;
+};
+
+ldaAgg: output lda of fv;
+
+visit(p, visitor {
+ # only look at the latest snapshot
+ before n: CodeRepository -> {
+ snapshot := getsnapshot(n);
+ foreach (i: int; def(snapshot[i]))
+ visit(snapshot[i]);
+ stop;
+ }
+ before node: Declaration -> {
+ if (node.kind == TypeKind.CLASS) {
+ public_ele : float = 0.0;
+ protected_ele : float = 0.0;
+ private_ele : float = 0.0;
+ other: float = 0.0;
+ foreach (i: int; def(node.methods[i])) {
+ if(has_modifier_public(node.methods[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.methods[i])) {
+ private_ele = private_ele + 1;
+ } else if(has_modifier_protected(node.methods[i])) {
+ protected_ele = protected_ele + 1;
+ } else {
+ other = other + 1;
+ }
+ }
+
+ foreach (i: int; def(node.fields[i])) {
+ if(has_modifier_public(node.fields[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.fields[i])) {
+ private_ele = private_ele + 1;
+ } else if(has_modifier_protected(node.fields[i])) {
+ protected_ele = protected_ele + 1;
+ }else {
+ other = other + 1;
+ }
+ }
+
+ feature1 : fv = {public_ele, private_ele, protected_ele, other};
+ ldaAgg << feature1;
+ ldaAgg << feature1;
+ ldaAgg << feature1;
+ ldaAgg << feature1;
+ ldaAgg << feature1;
+ ldaAgg << feature1;
+ ldaAgg << feature1;
+ ldaAgg << feature1;
+ }
+ }
+});
\ No newline at end of file
diff --git a/test/ml/demo_neural/neural.boa b/test/ml/demo_neural/neural.boa
new file mode 100644
index 000000000..662981499
--- /dev/null
+++ b/test/ml/demo_neural/neural.boa
@@ -0,0 +1,231 @@
+p: Project = input;
+testing : output sum of int;
+
+type Connection = {cweight : float, prevDeltaWeight : float, deltaWeight : float, leftNeuron : int, rightNeuron : int};
+type Neuron = {id: int, bias : float , outputVal : float, biasConnection : Connection, inConnection : array of Connection};
+type emitvals = {inp:array of float, expected: array of float};
+
+type Model = {inputL: array of Neuron, hiddenL: array of Neuron, outputL: array of Neuron, neuronMap : map[int] of Neuron};
+
+nueralNetworks := function(vals : array of emitvals, test123: float, test56: float) : Model {
+ runs : int = 500;
+ id_counter : int = 0;
+
+ dummyConnection: Connection = {-1.0, -1.0, -1.0, -1, -1};
+ dummyConnArray : array of Connection = {dummyConnection};
+
+ biasNeuron : Neuron = {id_counter, 0.0, 0.0, dummyConnection, dummyConnArray};
+ dummyArray : array of float = {0.0};
+ resultOutputs: array of array of float;
+ resultOutputs = new(resultOutputs, len(vals), dummyArray);
+
+ learningRate : float = 0.9;
+ momentum : float = 0.7;
+ layers : array of int = {2, 4, 1};
+ totalLayers := len(layers); # intputLayer = 0, hiddenLayer = 1; outputLayer = 2
+
+
+ inputLayer : array of Neuron = {biasNeuron, biasNeuron};
+ hiddenLayer : array of Neuron = {biasNeuron, biasNeuron, biasNeuron, biasNeuron};
+ outputLayer : array of Neuron = {biasNeuron};
+ neuronRecorder : map[int] of Neuron;
+
+ neuronRecorder[id_counter] = biasNeuron;
+ id_counter = id_counter + 1;
+ epsilon : float = 0.00000000001;
+
+
+ for(i :int = 1; i < totalLayers; i++) {
+ # input layer
+ if(i == 0) {
+ for(s :int = 0; s < layers[i]; s++) {
+ inputLayer[s] = {id_counter, 0.0, 0.0, dummyConnection, dummyConnArray};
+ neuronRecorder[id_counter] = inputLayer[s];
+ id_counter = id_counter + 1;
+ }
+ }
+ # hidden layer
+ if(i == 1) {
+ for(z :int = 0; z < layers[i]; z++) {
+ cons : array of Connection = {dummyConnection, dummyConnection};
+ #cons = new(cons, layers[0], dummyConnection);
+ node : Neuron = {id_counter, 0.0, 0.0, dummyConnection, cons};
+
+ # add connections
+ foreach(k: int; def(inputLayer[k])) {
+ localConnection: Connection = {rand(), 0.0, 0.0, inputLayer[k].id, node.id}; # assigns random connweight to the connection
+ node.inConnection[k] = localConnection;
+ }
+ neuronRecorder[id_counter] = node;
+ id_counter++;
+
+ #addInConnection(node, inputLayer);
+ hiddenLayer[z] = node;
+ }
+ }
+ # output layer
+ if(i == 2) {
+ for(j :int = 0; j < layers[i]; j++) {
+ cons1 : array of Connection = {dummyConnection, dummyConnection, dummyConnection, dummyConnection};
+ #cons1 = new(cons1, layers[1]);
+ node1 : Neuron = {id_counter, 0.0, 0.0, dummyConnection, cons1};
+
+ # add connections
+ foreach(k: int; def(hiddenLayer[k])) {
+ con1 : Connection = {rand(), 0.0, 0.0, hiddenLayer[k].id, node1.id}; # assigns random connweight to the connection
+ node1.inConnection[k] = con1;
+ }
+
+ neuronRecorder[id_counter] = node1;
+ id_counter++;
+ #addInConnection(node, hiddenLayer);
+ outputLayer[j] = node1;
+ }
+ }
+
+ }
+
+
+ error : float = 1.0;
+ for(m: int = 0; m < runs; m++) {
+ error = 0;
+ foreach(n : int; def(vals[n])) {
+ valueEmitted: emitvals = vals[n];
+ # set the input variables for jth value from values
+ foreach(k: int; def(inputLayer[k])) {
+ # there is one to one mapping in input neurons and number of features in each value
+ d: float = vals[n].inp[k];
+ inputLayer[k].outputVal = d;
+ }
+
+ # activate the neurons for the forward propagation
+ # calculate the output of each hiddenLayer Neuron
+ foreach(k : int; def(hiddenLayer[k])) {
+ node2: Neuron = hiddenLayer[k];
+ intermediateResult : float = 0.0;
+ connections :array of Connection = node2.inConnection;
+ foreach(l: int; def(connections[l])) {
+ left: Neuron = neuronRecorder[connections[l].leftNeuron];
+ connweight : float = connections[l].cweight;
+ intermediateResult = intermediateResult + (connweight * left.outputVal);
+ }
+ intermediateResult = intermediateResult + (node2.biasConnection.cweight * node2.bias);
+ node2.outputVal = 1.0 / (1.0 + exp(intermediateResult));
+ #calculateOutput(hiddenLayer[i]);
+ }
+ # calculate the output of each outputLayer Neuron
+ foreach(k : int; def(outputLayer[k])) {
+ node3:Neuron = outputLayer[k];
+ intermediateResult1 : float = 0.0;
+ connections1 :array of Connection = node3.inConnection;
+ foreach(l: int; def(connections1[l])) {
+ left1: Neuron = neuronRecorder[connections1[l].leftNeuron];
+ connweight1 : float = connections1[l].cweight;
+ intermediateResult1 = intermediateResult1 + (connweight1 * left1.outputVal);
+ }
+ intermediateResult1 = intermediateResult1 + (node3.biasConnection.cweight * node3.bias);
+ node3.outputVal = 1.0 / (1.0 + exp(intermediateResult1));
+ #calculateOutput(outputLayer[i]);
+ }
+
+ # output results of each loop
+ outputR : array of float;
+ outputR = new(outputR, len(outputLayer), 0);
+ foreach(l: int; def(outputLayer[l])) {
+ outputR[l] = outputLayer[l].outputVal;
+ }
+
+ resultOutputs[n] = outputR;
+
+ #calculate error
+ expectations :array of float = vals[n].expected;
+ foreach(l: int; def(expectations[l])) {
+ err : float = pow(outputR[l]- vals[n].expected[l], 2);
+ error = error + err;
+ }
+
+ # back propogration fpr expectations
+ #update output layer
+ outNodeCounter : int = 0;
+ foreach(o: int; def(outputLayer[o])) {
+ foreach(connNo: int; def(outputLayer[o].inConnection[connNo])) {
+ nodeConnection : Connection = outputLayer[o].inConnection[connNo];
+ ak: float = outputLayer[o].outputVal;
+ temp : Neuron = neuronRecorder[nodeConnection.leftNeuron];
+ ai: float = temp.outputVal;
+ desiredOut: float = expectations[outNodeCounter];
+ partialDerivative : float = (-1 * ak) * (1 - ak) * ai * (desiredOut - ak);
+ deltaWeightNew : float = (-1 * learningRate) * partialDerivative;
+ newWeight: float = nodeConnection.cweight + deltaWeightNew;
+ nodeConnection.deltaWeight = deltaWeightNew;
+ nodeConnection.cweight = newWeight + momentum * nodeConnection.prevDeltaWeight;
+ }
+ outNodeCounter = outNodeCounter + 1;
+ }
+
+
+ #update hidden layer
+ foreach(h: int; def(hiddenLayer[h])) {
+ hiddenconnections : array of Connection = hiddenLayer[h].inConnection;
+ foreach(connNo: int; def(hiddenconnections[connNo])) {
+ aj: float = hiddenLayer[h].outputVal;
+ targetNeuron : Neuron = neuronRecorder[hiddenconnections[connNo].leftNeuron];
+ aih: float = targetNeuron.outputVal;
+ sumKoutputs : float = 0.0;
+ jindex : int = 0;
+ foreach(o: int; def(outputLayer[o])) {
+ wjk : float = 0.0;
+ allInConns: array of Connection = outputLayer[o].inConnection;
+ foreach(conid: int; def(allInConns[conid])) {
+ if(allInConns[conid].leftNeuron == hiddenLayer[h].id) {
+ wjk = allInConns[conid].cweight;
+ }
+ }
+ desiredOutput: float = expectations[jindex];
+ akh:float = outputLayer[o].outputVal;
+ jindex = jindex + 1;
+ sumKoutputs = sumKoutputs + (-1 * (desiredOutput - akh) * akh * (1 - akh) * wjk);
+ }
+ partialDerivative_: float = aj * (1 - aj) * aih * sumKoutputs;
+ dWeight: float = (-1 * learningRate) * partialDerivative_;
+ newWeight1: float = hiddenconnections[connNo].cweight + dWeight;
+ hiddenconnections[connNo].deltaWeight = dWeight;
+ hiddenconnections[connNo].cweight = newWeight1+ momentum * hiddenconnections[connNo].prevDeltaWeight;
+ }
+ outNodeCounter = outNodeCounter + 1;
+ }
+
+ }
+ }
+ #return outputLayer[0].outputVal;
+ model : Model = {inputLayer, hiddenLayer, outputLayer, neuronRecorder};
+ return model;
+ };
+
+
+neuralNetwork : output nueralNetworks(0.001, 0.99) of emitvals;
+
+inps: array of float = {1.0, 1.0};
+out: array of float = {0.0};
+
+result: emitvals = {inps, out};
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
diff --git a/test/ml/demo_neural/neuralTest.boa b/test/ml/demo_neural/neuralTest.boa
new file mode 100644
index 000000000..a8e35ca0e
--- /dev/null
+++ b/test/ml/demo_neural/neuralTest.boa
@@ -0,0 +1,76 @@
+# What are the 5 largest projects, in terms of AST nodes?
+# Output is in Millions of AST nodes.
+p: Project = input;
+counter: output sum of int;
+
+
+type Connection = {cweight : float, prevDeltaWeight : float, deltaWeight : float, leftNeuron : int, rightNeuron : int};
+type Neuron = {id: int, bias : float , outputVal : float, biasConnection : Connection, inConnection : array of Connection};
+type emitvals = {inp:array of float, expected: array of float};
+type Model = {inputL: array of Neuron, hiddenL: array of Neuron, outputL: array of Neuron, neuronMap : map[int] of Neuron};
+
+
+############### dummy information about the model ###################
+
+dummyConnection: Connection = {-1.0, -1.0, -1.0, -1, -1};
+dummyConnArray : array of Connection = {dummyConnection};
+dummyNeuron : Neuron = {0, 0.0, 0.0, dummyConnection, dummyConnArray};
+dummyLayer : array of Neuron = {dummyNeuron};
+dummyMap: map[int] of Neuron;
+dummyModel : Model = {dummyLayer, dummyLayer, dummyLayer, dummyMap};
+
+############### dummy information ends ##############################
+
+results: output collection[string] of float;
+
+
+#loadedModel: Model = load("Neural.model", dummyModel);
+loadedModel: Model = load("./Neural/Neural.model", dummyModel);
+
+classification := function(model: Model, vals: array of float): float {
+ neuronRecorder : map[int] of Neuron = model.neuronMap;
+ # set the input variables for jth value from values
+ inputLayer := model.inputL;
+ outputLayer := model.outputL;
+ hiddenLayer := model.hiddenL;
+ foreach(k: int; def(inputLayer[k])) {
+ d: float = vals[k];
+ inputLayer[k].outputVal = d;
+ }
+
+# activate the neurons for the forward propagation
+ # calculate the output of each hiddenLayer Neuron
+ foreach(k : int; hiddenLayer[k]) {
+ node2: Neuron = hiddenLayer[k];
+ intermediateResult : float = 0.0;
+ connections :array of Connection = node2.inConnection;
+ foreach(l: int; def(connections[l])) {
+ left: Neuron = neuronRecorder[connections[l].leftNeuron];
+ connweight : float = connections[l].cweight;
+ intermediateResult = intermediateResult + (connweight * left.outputVal);
+ }
+ intermediateResult = intermediateResult + (node2.biasConnection.cweight * node2.bias);
+ node2.outputVal = 1.0 / (1.0 + exp(intermediateResult));
+ #calculateOutput(hiddenL[i]);
+ }
+ # calculate the output of each outputLayer Neuron
+ foreach(k : int; outputLayer[k]) {
+ node3:Neuron = outputLayer[k];
+ intermediateResult1 : float = 0.0;
+ connections1 :array of Connection = node3.inConnection;
+ foreach(l: int; def(connections1[l])) {
+ left1: Neuron = neuronRecorder[connections1[l].leftNeuron];
+ connweight1 : float = connections1[l].cweight;
+ intermediateResult1 = intermediateResult1 + (connweight1 * left1.outputVal);
+ }
+ intermediateResult1 = intermediateResult1 + (node3.biasConnection.cweight * node3.bias);
+ node3.outputVal = 1.0 / (1.0 + exp(intermediateResult1));
+ #calculateOutput(outputL[i]);
+ }
+ return outputLayer[0].outputVal;
+};
+
+#testInput : emitVals = {{0.0}, {1.1}};
+
+#results[p.id] << classification(loadedModel, testInput);
+results[p.id] << classification(loadedModel, {3.0, 0.3});
\ No newline at end of file
diff --git a/test/ml/lr.boa b/test/ml/lr.boa
new file mode 100644
index 000000000..8eddd728e
--- /dev/null
+++ b/test/ml/lr.boa
@@ -0,0 +1,93 @@
+# What are the 5 largest projects, in terms of AST nodes?
+# Output is in Millions of AST nodes.
+p: Project = input;
+counter: output sum[string][int] of int;
+collect3: output sum[int][int] of float;
+collect2: output sum[int][int] of float;
+collect4: output sum[int][int] of float;
+
+
+type fv = {pub:float, pri:float, pro:float, total:float};
+type sol = {pub_coff:float, pri_coff:float, pro_coff:float};
+
+
+
+lr := function(vals: array of fv): sol {
+ dataset : array of array of float = flattenedMatrix(vals, 4);
+ X : array of array of float = submatrix(dataset, 0, len(dataset)-1, 0, 2);
+ Y : array of array of float = transpose(getCol(dataset, 3));
+ X_trans : array of array of float = transpose(X);
+ A := inverse(multiply(X_trans, X));
+ B := multiply(X_trans, Y);
+ calculated : array of float = flatten(transpose(multiply(A, B)));
+ result : sol = {calculated[0], calculated[1], calculated[2]};
+ #result : sol = {len(X_trans)+0.0, len(X_trans[0])+ 0.0, len(Y) + 0.0};
+ return result;
+};
+
+lr_agg : output lr of fv;
+
+
+visit(p, visitor {
+ # only look at the latest snapshot
+ before n: CodeRepository -> {
+ snapshot := getsnapshot(n);
+ foreach (i: int; def(snapshot[i]))
+ visit(snapshot[i]);
+ stop;
+ }
+ before node: Declaration -> {
+ if (node.kind == TypeKind.CLASS) {
+ public_ele : int = 0;
+ protected_ele : int = 0;
+ private_ele : int = 0;
+ foreach (i: int; def(node.methods[i])) {
+ if(has_modifier_public(node.methods[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.methods[i])) {
+ private_ele = public_ele + 1;
+ } else {
+ protected_ele = protected_ele + 1;
+ }
+ }
+
+ foreach (i: int; def(node.fields[i])) {
+ if(has_modifier_public(node.fields[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.fields[i])) {
+ private_ele = public_ele + 1;
+ } else {
+ protected_ele = protected_ele + 1;
+ }
+ }
+
+ feature1 : fv = {public_ele + 0.0, private_ele + 0.0, protected_ele + 0.0, (public_ele + 0.0 + private_ele + protected_ele)};
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ lr_agg << feature1;
+ }
+ }
+});
+
diff --git a/test/ml/lrOptimized.boa b/test/ml/lrOptimized.boa
new file mode 100644
index 000000000..96891be2b
--- /dev/null
+++ b/test/ml/lrOptimized.boa
@@ -0,0 +1,86 @@
+# What are the 5 largest projects, in terms of AST nodes?
+# Output is in Millions of AST nodes.
+p: Project = input;
+counter: output sum of int;
+
+
+type fv = {pub:float, pri:float, pro:float};
+type mapped = {a: array of array of float, b: array of array of float};
+type sol = {pub_coff:float, pri_coff:float, pro_coff:float};
+
+
+
+lr := function(vals: array of mapped): sol {
+ result : sol;
+ #if (len(vals) > 0) {
+ vals_a :array of array of float = vals[0].a;
+ vals_b :array of array of float = vals[0].b;
+ foreach(i: int; def(vals[i])) {
+ vals_a = matrixsum(vals[i].a, vals_a);
+ vals_b = matrixsum(vals[i].b, vals_b);
+ }
+ vals_a = matrixsubstract(vals_a, vals[0].a);
+ vals_b = matrixsubstract(vals_b, vals[0].b);
+ calculated : array of float = flatten(transpose(multiply(inverse(vals_a), vals_b)));
+ result = {calculated[0], calculated[1], calculated[2]};
+ #}
+ return result;
+};
+
+lr_agg : output lr of fv;
+
+
+visit(p, visitor {
+ # only look at the latest snapshot
+ before n: CodeRepository -> {
+ snapshot := getsnapshot(n);
+ foreach (i: int; def(snapshot[i]))
+ visit(snapshot[i]);
+ stop;
+ }
+ before node: Declaration -> {
+ if (node.kind == TypeKind.CLASS) {
+ public_ele : int = 0;
+ protected_ele : int = 0;
+ private_ele : int = 0;
+ foreach (i: int; def(node.methods[i])) {
+ if(has_modifier_public(node.methods[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.methods[i])) {
+ private_ele = public_ele + 1;
+ } else {
+ protected_ele = protected_ele + 1;
+ }
+ }
+
+ foreach (i: int; def(node.fields[i])) {
+ if(has_modifier_public(node.fields[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.fields[i])) {
+ private_ele = public_ele + 1;
+ } else {
+ protected_ele = protected_ele + 1;
+ }
+ }
+
+ #feature1 : fv = {public_ele + 0.0, private_ele + 0.0, protected_ele + 0.0, (public_ele + 0.0 + private_ele + protected_ele)};
+ features : array of float = {public_ele + 0.0, private_ele + 0.0, protected_ele + 0.0};
+ label : array of float = {public_ele + 0.0 + private_ele + protected_ele};
+ X := {features};
+ Y := {label};
+ XTp := transpose(X);
+ cal1 : array of array of float = multiply(XTp, X);
+ cal2 : array of array of float = multiply(XTp, Y);
+ toEmit : mapped = {cal1, cal2};
+ count := 40;
+ while(count > 0) {
+ toEmit.a = matrixsum(toEmit.a, cal1);
+ toEmit.b = matrixsum(toEmit.b, cal2);
+ count = count - 1;
+ }
+ lr_agg << toEmit;
+ counter << 1;
+ }
+ }
+});
+
diff --git a/test/ml/naive.boa b/test/ml/naive.boa
new file mode 100644
index 000000000..53d78f143
--- /dev/null
+++ b/test/ml/naive.boa
@@ -0,0 +1,172 @@
+p: Project = input;
+type fv = {a:int, b:int, c:int, d:int};
+type stats = {a_stat:float, b_stat:float, c_stat:float};
+type complete_stat = {avg: stats, dev: stats};
+type Data = {training: fv, testing: fv};
+splitRatio : float = 0.67;
+
+naive := function(vals : array of Data) : float {
+ train : array of fv;
+ test : array of fv;
+
+ spearated: map[int] of array of fv; # classified per value
+ summaries : map[int] of complete_stat;
+
+ # separate the training and testing datasets
+ foreach(i:int; def(vals[i])) {
+ if(def(train)) {
+ train = train + {vals[i].training};
+ } else {
+ train = {vals[i].training};
+ }
+ if(def(test)) {
+ test = test+ {vals[i].testing};
+ } else {
+ test = {vals[i].testing};
+ }
+
+ }
+
+
+ # classify training datasets
+ foreach(i:int; def(train[i])) {
+ temp : array of fv = {train[i]};
+ if(!haskey(spearated, train[i].d)) {
+ spearated[train[i].d] = temp;
+ } else {
+ spearated[train[i].d] = spearated[train[i].d] + temp;
+ }
+ }
+
+ # all the classes
+ classes : array of int = keys(spearated);
+
+ # summarize data from training dataset
+ foreach(i:int; def(classes[i])) {
+ # calculate mean
+ feature_mean : stats = {0.0, 0.0, 0.0};
+ foreach(j:int; def(spearated[classes[i]][j])) {
+ feature_mean.a_stat = feature_mean.a_stat + spearated[classes[i]][j].a;
+ feature_mean.b_stat = feature_mean.b_stat + spearated[classes[i]][j].b;
+ feature_mean.c_stat = feature_mean.c_stat + spearated[classes[i]][j].c;
+ }
+ feature_mean.a_stat = feature_mean.a_stat / len(spearated[classes[i]]);
+ feature_mean.b_stat = feature_mean.b_stat / len(spearated[classes[i]]);
+ feature_mean.c_stat = feature_mean.c_stat / len(spearated[classes[i]]);
+
+
+ # calculate sd
+ feature_sd : stats = {0.0, 0.0, 0.0};
+ foreach(j:int; def(spearated[classes[i]][j])) {
+ feature_sd.a_stat = feature_sd.a_stat + (spearated[classes[i]][j].a - feature_mean.a_stat);
+ feature_sd.b_stat = feature_sd.b_stat + (spearated[classes[i]][j].b - feature_mean.b_stat);
+ feature_sd.c_stat = feature_sd.c_stat + (spearated[classes[i]][j].c - feature_mean.c_stat);
+ }
+ feature_sd.a_stat = sqrt(feature_sd.a_stat / len(spearated[classes[i]]));
+ feature_sd.b_stat = sqrt(feature_sd.b_stat / len(spearated[classes[i]]));
+ feature_sd.c_stat = sqrt(feature_sd.c_stat / len(spearated[classes[i]]));
+
+ # summarized a class
+ summaries[classes[i]] = {feature_mean, feature_sd};
+ }
+
+
+ predictions: array of int;
+ predictions = new(predictions, len(test), -1);
+
+ # predict for each test data
+ foreach(i:int; def(test[i])) {
+ probabilities : map[int] of float;
+ foreach(j: int; def(classes[j])) {
+ probabilities[classes[j]] = 1.0;
+ mean := summaries[classes[j]].avg;
+ deviation := summaries[classes[j]].dev;
+ probabilities[classes[j]] = probabilities[classes[j]] * (1/ (sqrt(2 * 3.14) * deviation.a_stat)) * (exp(-1 * ((pow((1.0 * test[i].a) - mean.a_stat, 2))/(2 * pow(deviation.a_stat, 2)))));
+ probabilities[classes[j]] = probabilities[classes[j]] * (1/ (sqrt(2 * 3.14) * deviation.a_stat)) * (exp(-1 * ((pow((1.0 * test[i].b) - mean.b_stat, 2))/(2 * pow(deviation.b_stat, 2)))));
+ probabilities[classes[j]] = probabilities[classes[j]] * (1/ (sqrt(2 * 3.14) * deviation.a_stat)) * (exp(-1 * ((pow((1.0 * test[i].c) - mean.c_stat, 2))/(2 * pow(deviation.c_stat, 2)))));
+ }
+
+ bestProb : float = 0;
+ bestLab : int = -1;
+ foreach(j: int; def(classes[j])) {
+ if ((bestLab == -1) || (bestProb < probabilities[classes[j]])) {
+ bestProb = probabilities[classes[j]];
+ bestLab = classes[j];
+ }
+ }
+ predictions[i] = bestLab;
+ }
+
+ correct : float = 0.0;
+ foreach(i:int; def(test[i])) {
+ if(predictions[i] == test[i].d) {
+ correct = correct + 1.0;
+ }
+ }
+ return correct/len(test) * 100;
+};
+
+scale := function(ast: int, method: int, class: int) : int {
+ total : int = 0;
+ if(ast > 1000) {
+ total++;
+ } if(method > 500) {
+ total++;
+ } if(class > 50) {
+ total++;
+ }
+ return total;
+};
+
+
+naive_bayes : output naive of Data;
+
+# count ast nodes
+
+astCount := 0;
+classCount := 0;
+methodCount := 0;
+visit(p, visitor {
+ # only look at the latest snapshot
+ before n: CodeRepository -> {
+ snapshot := getsnapshot(n);
+ foreach (i: int; def(snapshot[i]))
+ visit(snapshot[i]);
+ stop;
+ }
+ before node: Declaration -> {
+ if (node.kind == TypeKind.CLASS) {
+ classCount++;
+ foreach (i: int; node.methods[i]) {
+ methodCount++;
+ }
+ }
+ }
+ # by default, count all visited nodes
+ before _ -> astCount++;
+ # these nodes are not part of the AST, so do nothing when visiting
+ before Project, ChangedFile -> ;
+});
+
+
+
+dummy : fv = {0, 0, 0, 0};
+nondummy : fv = {astCount, methodCount, classCount, scale(astCount, methodCount, classCount)};
+data1: Data = {nondummy, dummy};
+data2: Data = {dummy, nondummy};
+if(rand() > splitRatio)
+ naive_bayes << data1;
+else
+ naive_bayes << data2;
+
+
+if(rand() > splitRatio)
+ naive_bayes << data1;
+else
+ naive_bayes << data2;
+
+
+if(rand() > splitRatio)
+ naive_bayes << data1;
+else
+ naive_bayes << data2;
diff --git a/test/ml/neural.boa b/test/ml/neural.boa
new file mode 100644
index 000000000..b68d0b8a4
--- /dev/null
+++ b/test/ml/neural.boa
@@ -0,0 +1,232 @@
+p: Project = input;
+testing : output sum of int;
+
+type Connection = {cweight : float, prevDeltaWeight : float, deltaWeight : float, leftNeuron : int, rightNeuron : int};
+type Neuron = {id: int, bias : float , outputVal : float, biasConnection : Connection, inConnection : array of Connection};
+type emitvals = {inp:array of float, expected: array of float};
+testing << 1;
+
+nueralNetworks := function(vals : array of emitvals) :float {
+ runs : int = 500;
+ thresholdError := 0.001;
+ id_counter : int = 0;
+
+ dummyConnection: Connection = {-1.0, -1.0, -1.0, -1, -1};
+ dummyConnArray : array of Connection = {dummyConnection};
+
+ biasNeuron : Neuron = {id_counter, 0.0, 0.0, dummyConnection, dummyConnArray};
+ dummyArray : array of float = {0.0};
+ resultOutputs: array of array of float;
+ resultOutputs = new(resultOutputs, len(vals), dummyArray);
+
+ learningRate : float = 0.9;
+ momentum : float = 0.7;
+ layers : array of int = {2, 4, 1};
+ totalLayers := len(layers); # intputLayer = 0, hiddenLayer = 1; outputLayer = 2
+
+
+ inputLayer : array of Neuron = {biasNeuron, biasNeuron};
+ hiddenLayer : array of Neuron = {biasNeuron, biasNeuron, biasNeuron, biasNeuron};
+ outputLayer : array of Neuron = {biasNeuron};
+ neuronRecorder : map[int] of Neuron;
+
+ neuronRecorder[id_counter] = biasNeuron;
+ id_counter = id_counter + 1;
+ epsilon : float = 0.00000000001;
+
+
+ for(i :int = 1; i < totalLayers; i++) {
+ # input layer
+ if(i == 0) {
+ for(s :int = 0; s < layers[i]; s++) {
+ inputLayer[s] = {id_counter, 0.0, 0.0, dummyConnection, dummyConnArray};
+ neuronRecorder[id_counter] = inputLayer[s];
+ id_counter = id_counter + 1;
+ }
+ }
+ # hidden layer
+ if(i == 1) {
+ for(z :int = 0; z < layers[i]; z++) {
+ cons : array of Connection = {dummyConnection, dummyConnection};
+ #cons = new(cons, layers[0], dummyConnection);
+ node : Neuron = {id_counter, 0.0, 0.0, dummyConnection, cons};
+
+ # add connections
+ foreach(k: int; def(inputLayer[k])) {
+ localConnection: Connection = {rand(), 0.0, 0.0, inputLayer[k].id, node.id}; # assigns random connweight to the connection
+ node.inConnection[k] = localConnection;
+ }
+ neuronRecorder[id_counter] = node;
+ id_counter++;
+
+ #addInConnection(node, inputLayer);
+ hiddenLayer[z] = node;
+ }
+ }
+ # output layer
+ if(i == 2) {
+ for(j :int = 0; j < layers[i]; j++) {
+ cons1 : array of Connection = {dummyConnection, dummyConnection, dummyConnection, dummyConnection};
+ #cons1 = new(cons1, layers[1]);
+ node1 : Neuron = {id_counter, 0.0, 0.0, dummyConnection, cons1};
+
+ # add connections
+ foreach(k: int; def(hiddenLayer[k])) {
+ con1 : Connection = {rand(), 0.0, 0.0, hiddenLayer[k].id, node1.id}; # assigns random connweight to the connection
+ node1.inConnection[k] = con1;
+ }
+
+ neuronRecorder[id_counter] = node1;
+ id_counter++;
+ #addInConnection(node, hiddenLayer);
+ outputLayer[j] = node1;
+ }
+ }
+
+ }
+
+
+ error : float = 1.0;
+ for(m: int = 0; m < runs; m++) {
+ error = 0;
+ foreach(n : int; def(vals[n])) {
+ valueEmitted: emitvals = vals[n];
+ # set the input variables for jth value from values
+ foreach(k: int; def(inputLayer[k])) {
+ # there is one to one mapping in input neurons and number of features in each value
+ print("m");
+ print(m); print("k");print(k);
+ d: float = vals[n].inp[k];
+ inputLayer[k].outputVal = d;
+ }
+
+ # activate the neurons for the forward propagation
+ # calculate the output of each hiddenLayer Neuron
+ foreach(k : int; def(hiddenLayer[k])) {
+ node2: Neuron = hiddenLayer[k];
+ intermediateResult : float = 0.0;
+ connections :array of Connection = node2.inConnection;
+ foreach(l: int; def(connections[l])) {
+ print(neuronRecorder);
+ printany(connections[l].leftNeuron);
+ left: Neuron = neuronRecorder[connections[l].leftNeuron];
+ connweight : float = connections[l].cweight;
+ intermediateResult = intermediateResult + (connweight * left.outputVal);
+ }
+ intermediateResult = intermediateResult + (node2.biasConnection.cweight * node2.bias);
+ node2.outputVal = 1.0 / (1.0 + exp(intermediateResult));
+ #calculateOutput(hiddenLayer[i]);
+ }
+ # calculate the output of each outputLayer Neuron
+ foreach(k : int; def(outputLayer[k])) {
+ node3:Neuron = outputLayer[k];
+ intermediateResult1 : float = 0.0;
+ connections1 :array of Connection = node3.inConnection;
+ foreach(l: int; def(connections1[l])) {
+ left1: Neuron = neuronRecorder[connections1[l].leftNeuron];
+ connweight1 : float = connections1[l].cweight;
+ intermediateResult1 = intermediateResult1 + (connweight1 * left1.outputVal);
+ }
+ intermediateResult1 = intermediateResult1 + (node3.biasConnection.cweight * node3.bias);
+ node3.outputVal = 1.0 / (1.0 + exp(intermediateResult1));
+ #calculateOutput(outputLayer[i]);
+ }
+
+ # output results of each loop
+ outputR : array of float;
+ outputR = new(outputR, len(outputLayer), 0);
+ foreach(l: int; def(outputLayer[l])) {
+ outputR[l] = outputLayer[l].outputVal;
+ }
+
+ resultOutputs[n] = outputR;
+
+ #calculate error
+ expectations :array of float = vals[n].expected;
+ foreach(l: int; def(expectations[l])) {
+ err : float = pow(outputR[l]- vals[n].expected[l], 2);
+ error = error + err;
+ }
+
+ # back propogration fpr expectations
+ #update output layer
+ outNodeCounter : int = 0;
+ foreach(o: int; def(outputLayer[o])) {
+ foreach(connNo: int; def(outputLayer[o].inConnection[connNo])) {
+ nodeConnection : Connection = outputLayer[o].inConnection[connNo];
+ ak: float = outputLayer[o].outputVal;
+ temp : Neuron = neuronRecorder[nodeConnection.leftNeuron];
+ ai: float = temp.outputVal;
+ desiredOut: float = expectations[outNodeCounter];
+ partialDerivative : float = (-1 * ak) * (1 - ak) * ai * (desiredOut - ak);
+ deltaWeightNew : float = (-1 * learningRate) * partialDerivative;
+ newWeight: float = nodeConnection.cweight + deltaWeightNew;
+ nodeConnection.deltaWeight = deltaWeightNew;
+ nodeConnection.cweight = newWeight + momentum * nodeConnection.prevDeltaWeight;
+ }
+ outNodeCounter = outNodeCounter + 1;
+ }
+
+
+ #update hidden layer
+ foreach(h: int; def(hiddenLayer[h])) {
+ hiddenconnections : array of Connection = hiddenLayer[h].inConnection;
+ foreach(connNo: int; def(hiddenconnections[connNo])) {
+ aj: float = hiddenLayer[h].outputVal;
+ targetNeuron : Neuron = neuronRecorder[hiddenconnections[connNo].leftNeuron];
+ aih: float = targetNeuron.outputVal;
+ sumKoutputs : float = 0.0;
+ jindex : int = 0;
+ foreach(o: int; def(outputLayer[o])) {
+ wjk : float = 0.0;
+ allInConns: array of Connection = outputLayer[o].inConnection;
+ foreach(conid: int; def(allInConns[conid])) {
+ if(allInConns[conid].leftNeuron == hiddenLayer[h].id) {
+ wjk = allInConns[conid].cweight;
+ }
+ }
+ desiredOutput: float = expectations[jindex];
+ akh:float = outputLayer[o].outputVal;
+ jindex = jindex + 1;
+ sumKoutputs = sumKoutputs + (-1 * (desiredOutput - akh) * akh * (1 - akh) * wjk);
+ }
+ partialDerivative_: float = aj * (1 - aj) * aih * sumKoutputs;
+ dWeight: float = (-1 * learningRate) * partialDerivative_;
+ newWeight1: float = hiddenconnections[connNo].cweight + dWeight;
+ hiddenconnections[connNo].deltaWeight = dWeight;
+ hiddenconnections[connNo].cweight = newWeight1+ momentum * hiddenconnections[connNo].prevDeltaWeight;
+ }
+ outNodeCounter = outNodeCounter + 1;
+ }
+
+ }
+ }
+ return outputLayer[0].outputVal;
+ };
+
+
+neuralNetwork : output nueralNetworks of emitvals;
+
+inps: array of float = {1.0, 1.0};
+out: array of float = {0.0};
+
+result: emitvals = {inps, out};
+neuralNetwork << result;
+
+inps = {1.0, 0.0};
+out = {1.0};
+
+result= {inps, out};
+neuralNetwork << result;
+
+inps = {0.0, 1.0};
+out = {1.0};
+
+result = {inps, out};
+neuralNetwork << result;
+
+inps = {0.0, 0.0};
+out = {0.0};
+
+result = {inps, out};
+neuralNetwork << result;
diff --git a/test/ml/neural_networks_training_testing/neural_networks_testing.boa b/test/ml/neural_networks_training_testing/neural_networks_testing.boa
new file mode 100644
index 000000000..0d6cdf370
--- /dev/null
+++ b/test/ml/neural_networks_training_testing/neural_networks_testing.boa
@@ -0,0 +1,37 @@
+# What are the 5 largest projects, in terms of AST nodes?
+# Output is in Millions of AST nodes.
+p: Project = input;
+counter: output sum of int;
+
+
+type Connection = {cweight : float, prevDeltaWeight : float, deltaWeight : float, leftNeuron : int, rightNeuron : int};
+type Neuron = {id: int, bias : float , outputVal : float, biasConnection : Connection, inConnection : array of Connection};
+type emitvals = {inp:array of float, expected: array of float};
+type Model = {inputL: array of Neuron, hiddenL: array of Neuron, outputL: array of Neuron};
+
+
+############### dummy information about the model ###################
+
+dummyConnection: Connection = {-1.0, -1.0, -1.0, -1, -1};
+dummyConnArray : array of Connection = {dummyConnection};
+dummyNeuron : Neuron = {0, 0.0, 0.0, dummyConnection, dummyConnArray};
+dummyLayer : array of Neuron = {dummyNeuron};
+dummyModel : Model = {dummyLayer, dummyLayer, dummyLayer};
+
+############### dummy information ends ##############################
+
+results: output collection[string] of float;
+
+
+#loadedModel: Model = load("Neural.model", dummyModel);
+loadedModel: Model = load("./Neural/Neural.model", dummyModel);
+
+classification := function(model: Model, testData: array of float): float {
+ outNeuron: Neuron = model.outputL[0];
+ return outNeuron.outputVal;
+};
+
+#testInput : emitVals = {{0.0}, {1.1}};
+
+#results[p.id] << classification(loadedModel, testInput);
+results[p.id] << classification(loadedModel, {0.0, 0.0});
\ No newline at end of file
diff --git a/test/ml/neural_networks_training_testing/neural_networks_train.boa b/test/ml/neural_networks_training_testing/neural_networks_train.boa
new file mode 100644
index 000000000..a8decbd6b
--- /dev/null
+++ b/test/ml/neural_networks_training_testing/neural_networks_train.boa
@@ -0,0 +1,231 @@
+p: Project = input;
+testing : output sum of int;
+
+type Connection = {cweight : float, prevDeltaWeight : float, deltaWeight : float, leftNeuron : int, rightNeuron : int};
+type Neuron = {id: int, bias : float , outputVal : float, biasConnection : Connection, inConnection : array of Connection};
+type emitvals = {inp:array of float, expected: array of float};
+
+type Model = {inputL: array of Neuron, hiddenL: array of Neuron, outputL: array of Neuron};
+
+nueralNetworks := function(vals : array of emitvals) : Model {
+ runs : int = 500;
+ id_counter : int = 0;
+
+ dummyConnection: Connection = {-1.0, -1.0, -1.0, -1, -1};
+ dummyConnArray : array of Connection = {dummyConnection};
+
+ biasNeuron : Neuron = {id_counter, 0.0, 0.0, dummyConnection, dummyConnArray};
+ dummyArray : array of float = {0.0};
+ resultOutputs: array of array of float;
+ resultOutputs = new(resultOutputs, len(vals), dummyArray);
+
+ learningRate : float = 0.9;
+ momentum : float = 0.7;
+ layers : array of int = {2, 4, 1};
+ totalLayers := len(layers); # intputLayer = 0, hiddenLayer = 1; outputLayer = 2
+
+
+ inputLayer : array of Neuron = {biasNeuron, biasNeuron};
+ hiddenLayer : array of Neuron = {biasNeuron, biasNeuron, biasNeuron, biasNeuron};
+ outputLayer : array of Neuron = {biasNeuron};
+ neuronRecorder : map[int] of Neuron;
+
+ neuronRecorder[id_counter] = biasNeuron;
+ id_counter = id_counter + 1;
+ epsilon : float = 0.00000000001;
+
+
+ for(i :int = 1; i < totalLayers; i++) {
+ # input layer
+ if(i == 0) {
+ for(s :int = 0; s < layers[i]; s++) {
+ inputLayer[s] = {id_counter, 0.0, 0.0, dummyConnection, dummyConnArray};
+ neuronRecorder[id_counter] = inputLayer[s];
+ id_counter = id_counter + 1;
+ }
+ }
+ # hidden layer
+ if(i == 1) {
+ for(z :int = 0; z < layers[i]; z++) {
+ cons : array of Connection = {dummyConnection, dummyConnection};
+ #cons = new(cons, layers[0], dummyConnection);
+ node : Neuron = {id_counter, 0.0, 0.0, dummyConnection, cons};
+
+ # add connections
+ foreach(k: int; def(inputLayer[k])) {
+ localConnection: Connection = {rand(), 0.0, 0.0, inputLayer[k].id, node.id}; # assigns random connweight to the connection
+ node.inConnection[k] = localConnection;
+ }
+ neuronRecorder[id_counter] = node;
+ id_counter++;
+
+ #addInConnection(node, inputLayer);
+ hiddenLayer[z] = node;
+ }
+ }
+ # output layer
+ if(i == 2) {
+ for(j :int = 0; j < layers[i]; j++) {
+ cons1 : array of Connection = {dummyConnection, dummyConnection, dummyConnection, dummyConnection};
+ #cons1 = new(cons1, layers[1]);
+ node1 : Neuron = {id_counter, 0.0, 0.0, dummyConnection, cons1};
+
+ # add connections
+ foreach(k: int; def(hiddenLayer[k])) {
+ con1 : Connection = {rand(), 0.0, 0.0, hiddenLayer[k].id, node1.id}; # assigns random connweight to the connection
+ node1.inConnection[k] = con1;
+ }
+
+ neuronRecorder[id_counter] = node1;
+ id_counter++;
+ #addInConnection(node, hiddenLayer);
+ outputLayer[j] = node1;
+ }
+ }
+
+ }
+
+
+ error : float = 1.0;
+ for(m: int = 0; m < runs; m++) {
+ error = 0;
+ foreach(n : int; def(vals[n])) {
+ valueEmitted: emitvals = vals[n];
+ # set the input variables for jth value from values
+ foreach(k: int; def(inputLayer[k])) {
+ # there is one to one mapping in input neurons and number of features in each value
+ d: float = vals[n].inp[k];
+ inputLayer[k].outputVal = d;
+ }
+
+ # activate the neurons for the forward propagation
+ # calculate the output of each hiddenLayer Neuron
+ foreach(k : int; def(hiddenLayer[k])) {
+ node2: Neuron = hiddenLayer[k];
+ intermediateResult : float = 0.0;
+ connections :array of Connection = node2.inConnection;
+ foreach(l: int; def(connections[l])) {
+ left: Neuron = neuronRecorder[connections[l].leftNeuron];
+ connweight : float = connections[l].cweight;
+ intermediateResult = intermediateResult + (connweight * left.outputVal);
+ }
+ intermediateResult = intermediateResult + (node2.biasConnection.cweight * node2.bias);
+ node2.outputVal = 1.0 / (1.0 + exp(intermediateResult));
+ #calculateOutput(hiddenLayer[i]);
+ }
+ # calculate the output of each outputLayer Neuron
+ foreach(k : int; def(outputLayer[k])) {
+ node3:Neuron = outputLayer[k];
+ intermediateResult1 : float = 0.0;
+ connections1 :array of Connection = node3.inConnection;
+ foreach(l: int; def(connections1[l])) {
+ left1: Neuron = neuronRecorder[connections1[l].leftNeuron];
+ connweight1 : float = connections1[l].cweight;
+ intermediateResult1 = intermediateResult1 + (connweight1 * left1.outputVal);
+ }
+ intermediateResult1 = intermediateResult1 + (node3.biasConnection.cweight * node3.bias);
+ node3.outputVal = 1.0 / (1.0 + exp(intermediateResult1));
+ #calculateOutput(outputLayer[i]);
+ }
+
+ # output results of each loop
+ outputR : array of float;
+ outputR = new(outputR, len(outputLayer), 0);
+ foreach(l: int; def(outputLayer[l])) {
+ outputR[l] = outputLayer[l].outputVal;
+ }
+
+ resultOutputs[n] = outputR;
+
+ #calculate error
+ expectations :array of float = vals[n].expected;
+ foreach(l: int; def(expectations[l])) {
+ err : float = pow(outputR[l]- vals[n].expected[l], 2);
+ error = error + err;
+ }
+
+ # back propogration fpr expectations
+ #update output layer
+ outNodeCounter : int = 0;
+ foreach(o: int; def(outputLayer[o])) {
+ foreach(connNo: int; def(outputLayer[o].inConnection[connNo])) {
+ nodeConnection : Connection = outputLayer[o].inConnection[connNo];
+ ak: float = outputLayer[o].outputVal;
+ temp : Neuron = neuronRecorder[nodeConnection.leftNeuron];
+ ai: float = temp.outputVal;
+ desiredOut: float = expectations[outNodeCounter];
+ partialDerivative : float = (-1 * ak) * (1 - ak) * ai * (desiredOut - ak);
+ deltaWeightNew : float = (-1 * learningRate) * partialDerivative;
+ newWeight: float = nodeConnection.cweight + deltaWeightNew;
+ nodeConnection.deltaWeight = deltaWeightNew;
+ nodeConnection.cweight = newWeight + momentum * nodeConnection.prevDeltaWeight;
+ }
+ outNodeCounter = outNodeCounter + 1;
+ }
+
+
+ #update hidden layer
+ foreach(h: int; def(hiddenLayer[h])) {
+ hiddenconnections : array of Connection = hiddenLayer[h].inConnection;
+ foreach(connNo: int; def(hiddenconnections[connNo])) {
+ aj: float = hiddenLayer[h].outputVal;
+ targetNeuron : Neuron = neuronRecorder[hiddenconnections[connNo].leftNeuron];
+ aih: float = targetNeuron.outputVal;
+ sumKoutputs : float = 0.0;
+ jindex : int = 0;
+ foreach(o: int; def(outputLayer[o])) {
+ wjk : float = 0.0;
+ allInConns: array of Connection = outputLayer[o].inConnection;
+ foreach(conid: int; def(allInConns[conid])) {
+ if(allInConns[conid].leftNeuron == hiddenLayer[h].id) {
+ wjk = allInConns[conid].cweight;
+ }
+ }
+ desiredOutput: float = expectations[jindex];
+ akh:float = outputLayer[o].outputVal;
+ jindex = jindex + 1;
+ sumKoutputs = sumKoutputs + (-1 * (desiredOutput - akh) * akh * (1 - akh) * wjk);
+ }
+ partialDerivative_: float = aj * (1 - aj) * aih * sumKoutputs;
+ dWeight: float = (-1 * learningRate) * partialDerivative_;
+ newWeight1: float = hiddenconnections[connNo].cweight + dWeight;
+ hiddenconnections[connNo].deltaWeight = dWeight;
+ hiddenconnections[connNo].cweight = newWeight1+ momentum * hiddenconnections[connNo].prevDeltaWeight;
+ }
+ outNodeCounter = outNodeCounter + 1;
+ }
+
+ }
+ }
+ #return outputLayer[0].outputVal;
+ model : Model = {inputLayer, hiddenLayer, outputLayer};
+ return model;
+ };
+
+
+neuralNetwork : output nueralNetworks(thresholdError : float = 0.001) of emitvals;
+
+inps: array of float = {1.0, 1.0};
+out: array of float = {0.0};
+
+result: emitvals = {inps, out};
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
+neuralNetwork << result;
diff --git a/test/ml/pca.boa b/test/ml/pca.boa
new file mode 100644
index 000000000..e92b70a21
--- /dev/null
+++ b/test/ml/pca.boa
@@ -0,0 +1,71 @@
+# Counting the 10 most used programming languages
+p: Project = input;
+counts: output top(10) of string weight int;
+astCount: int = 0;
+type fv = {a:float, b:float, c:float, d:float};
+
+
+pca:= function(vals: array of fv): fv {
+ meanarray: array of float = means(vals);
+ dataset : array of array of float = meanScaling(flattenedMatrix(vals, 4), meanarray);
+ meanvector: array of array of float = vector(meanarray);
+ parta := multiply(transpose(dataset), dataset);
+ meansResults := multiply(meanvector, meanarray);
+ cov := matrixsubstract(parta, meansResults);
+ eigens: array of float = eigenvalsReal(cov);
+ result : fv = {eigens[0], eigens[1], eigens[2], eigens[3]};
+ return result;
+};
+
+pcsAgg: output pca of fv;
+
+visit(p, visitor {
+ # only look at the latest snapshot
+ before n: CodeRepository -> {
+ snapshot := getsnapshot(n);
+ foreach (i: int; def(snapshot[i]))
+ visit(snapshot[i]);
+ stop;
+ }
+ before node: Declaration -> {
+ if (node.kind == TypeKind.CLASS) {
+ public_ele : float = 0.0;
+ protected_ele : float = 0.0;
+ private_ele : float = 0.0;
+ other: float = 0.0;
+ foreach (i: int; def(node.methods[i])) {
+ if(has_modifier_public(node.methods[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.methods[i])) {
+ private_ele = private_ele + 1;
+ } else if(has_modifier_protected(node.methods[i])) {
+ protected_ele = protected_ele + 1;
+ } else {
+ other = other + 1;
+ }
+ }
+
+ foreach (i: int; def(node.fields[i])) {
+ if(has_modifier_public(node.fields[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.fields[i])) {
+ private_ele = private_ele + 1;
+ } else if(has_modifier_protected(node.fields[i])) {
+ protected_ele = protected_ele + 1;
+ }else {
+ other = other + 1;
+ }
+ }
+
+ feature1 : fv = {public_ele, private_ele, protected_ele, other};
+ pcsAgg << feature1;
+ pcsAgg << feature1;
+ pcsAgg << feature1;
+ pcsAgg << feature1;
+ pcsAgg << feature1;
+ pcsAgg << feature1;
+ pcsAgg << feature1;
+ pcsAgg << feature1;
+ }
+ }
+});
\ No newline at end of file
diff --git a/test/ml/pcaOptimize.boa b/test/ml/pcaOptimize.boa
new file mode 100644
index 000000000..41b77ac44
--- /dev/null
+++ b/test/ml/pcaOptimize.boa
@@ -0,0 +1,100 @@
+# Counting the 10 most used programming languages
+p: Project = input;
+counts: output top(10) of string weight int;
+astCount: int = 0;
+type sol = {eigenvals: array of float, eigenvectors: array of array of float};
+type optimizedfv = {vectormult: array of array of float, featureMeans: array of float, totalInstances: int};
+
+mapperLocalTotal: array of float;
+mapperLocalTotal = new(mapperLocalTotal, 4, 0.0);
+mapperLocalFeature : array of array of float;
+mapperLocalFeature = new(mapperLocalFeature, 4, mapperLocalTotal);
+mapperLocalCounter: int = 0;
+
+
+pca:= function(vals: array of optimizedfv): sol {
+ parta :array of array of float = vals[0].vectormult;
+ meansOfFeature: array of float = multiply(vals[0].featureMeans, vals[0].totalInstances);
+ accumultedInstances: int = vals[0].totalInstances;
+
+ for(i:int = 1; i < len(vals); i++) {
+ parta = matrixsum(parta, vals[i].vectormult);
+ tempFeatures: array of float = multiply(vals[i].featureMeans, vals[i].totalInstances);
+ meansOfFeature = matrixsum(meansOfFeature, tempFeatures);
+ accumultedInstances = accumultedInstances + vals[i].totalInstances;
+ }
+
+ # adjust means now
+ meansOfFeature[0] = meansOfFeature[0] / accumultedInstances;
+ meansOfFeature[1] = meansOfFeature[1] / accumultedInstances;
+ meansOfFeature[2] = meansOfFeature[2] / accumultedInstances;
+ meansOfFeature[3] = meansOfFeature[3] / accumultedInstances;
+
+ partb : array of array of float = multiply(vector(meansOfFeature), meansOfFeature);
+
+ cov := matrixsubstract(parta, partb);
+ eigens: array of float = eigenvalsReal(cov);
+ #result : fv = {eigens[0], eigens[1], eigens[2], eigens[3]};
+ result : sol = {eigens, eigenvectors(cov)};
+ return result;
+};
+
+
+pcsAgg: output pca of optimizedfv;
+
+visit(p, visitor {
+ # only look at the latest snapshot
+ before n: CodeRepository -> {
+ snapshot := getsnapshot(n);
+ foreach (i: int; def(snapshot[i]))
+ visit(snapshot[i]);
+ stop;
+ }
+ before node: Declaration -> {
+ if (node.kind == TypeKind.CLASS) {
+ public_ele : float = 0.0;
+ protected_ele : float = 0.0;
+ private_ele : float = 0.0;
+ other: float = 0.0;
+ foreach (i: int; def(node.methods[i])) {
+ if(has_modifier_public(node.methods[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.methods[i])) {
+ private_ele = private_ele + 1;
+ } else if(has_modifier_protected(node.methods[i])) {
+ protected_ele = protected_ele + 1;
+ } else {
+ other = other + 1;
+ }
+ }
+
+ foreach (i: int; def(node.fields[i])) {
+ if(has_modifier_public(node.fields[i])) {
+ public_ele = public_ele + 1;
+ } else if(has_modifier_private(node.fields[i])) {
+ private_ele = private_ele + 1;
+ } else if(has_modifier_protected(node.fields[i])) {
+ protected_ele = protected_ele + 1;
+ }else {
+ other = other + 1;
+ }
+ }
+
+ meanPart : array of float = {public_ele, private_ele, protected_ele, other};
+ mapperLocalFeature = matrixsum(mapperLocalFeature, multiply(vector(meanPart), meanPart));
+ mapperLocalTotal[0] = mapperLocalTotal[0] + public_ele;
+ mapperLocalTotal[1] = mapperLocalTotal[1] + private_ele;
+ mapperLocalTotal[2] = mapperLocalTotal[2] + protected_ele;
+ mapperLocalTotal[3] = mapperLocalTotal[3] + other;
+ mapperLocalCounter = mapperLocalCounter + 1;
+ }
+ }
+});
+
+mapperLocalTotal[0] = mapperLocalTotal[0] / mapperLocalCounter;
+mapperLocalTotal[1] = mapperLocalTotal[1] / mapperLocalCounter;
+mapperLocalTotal[2] = mapperLocalTotal[2] / mapperLocalCounter;
+mapperLocalTotal[3] = mapperLocalTotal[3] / mapperLocalCounter;
+
+feature1: optimizedfv = {mapperLocalFeature, mapperLocalTotal, mapperLocalCounter};
+pcsAgg << feature1;
\ No newline at end of file