Import of FM index application

applications/fm-index/Readme.md

@@ -0,0 +1 @@
+Test data available from: http://www.doc.ic.ac.uk/~pg1709/index.tar.gz.

applications/fm-index/build/Makefile

@@ -0,0 +1,100 @@
+# Makefile ----- James Arram 2014
+
+# DFE parameters
+DFEModel=MAIA
+MPCX=true
+DEVICENUM=MAIA
+SLIC_CONF="default_engine_resource=192.168.0.12;default_wfi_timeout=60;default_pcie_timeout=60"
+
+# project name
+PRJ=align
+
+# design names
+D1=Em
+
+# paths
+D1SIMDIR=$(D1)_$(DFEModel)_DFE_SIM/results
+D1DFEDIR=$(D1)_$(DFEModel)_DFE/results
+MAXELEROSDIR_SIM:=$(MAXCOMPILERDIR)/lib/maxeleros-sim
+SRCDIR=../src
+OBJDIR=obj
+
+# source files
+D1FILES=$(wildcard $(SRCDIR)/*.maxj)
+CPUFILES=$(wildcard $(SRCDIR)/*.cpp)
+DEPFILES=$(wildcard $(SRCDIR)/*.hpp)
+OBJFILES=$(patsubst $(SRCDIR)/%.cpp, obj/%.o, $(CPUFILES))
+
+# compiler 
+CC=g++
+MAXFILECOMPILE=maxfilecompile
+SLICCOMPILE=sliccompile
+MAXGUESSBUILDDIR=maxGuessBuildDir
+MAXFILESTITCH=maxfilestitch
+MAXJAVARUN=maxJavaRun
+MAXJC=maxjc
+MAXDEBUG=maxdebug
+MAXRENDERGRAPHS=maxRenderGraphs
+MAXCOMPILERSIM=maxcompilersim
+
+JFLAGS=-cp $(MAXCOMPILERDIR)/lib/MaxCompiler.jar -1.6 -d .
+CFLAGS=-O3 -g -fopenmp -Wall -Wextra -I${MAXCOMPILERDIR}/include -I${MAXCOMPILERDIR}/include/slic -I${MAXELER\OSDIR}/include_
+LFLAGS=-L${MAXCOMPILERDIR}/lib -L${MAXELEROSDIR}/lib -lmaxeleros -lslic -lm -lpthread -fopenmp
+
+# simulation
+$(D1SIMDIR)/$(D1).max: $(D1FILES)
+	$(MAXJC) $(JFLAGS) $(D1FILES)
+	MAXAPPJCP=. MAXSOURCEDIRS='../src' $(MAXJAVARUN) -v -m 8192 $(D1)Manager DFEModel=$(DFEModel) maxFileName=$(D1) target='DFE_SIM' enableMPCX=$(MPCX)
+	cp $(D1SIMDIR)/$(D1).h $(D1SIMDIR)/Maxfiles.h
+	@ mkdir -p class
+	mv *.class class/
+
+$(OBJDIR)/$(D1)_sim.o: $(D1SIMDIR)/$(D1).max
+	@ mkdir -p obj
+	$(SLICCOMPILE) $< $@
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.cpp $(DEPFILES)
+	@ mkdir -p obj
+	$(CC) $< $(CFLAGS) -I$(D1SIMDIR) -DDESIGN_NAME=$(PRJ) -c -o $@
+
+$(PRJ)_sim: $(OBJDIR)/$(D1)_sim.o $(OBJFILES)
+	$(CC) $(CFLAGS) -o $@ $^ $(LFLAGS)
+
+runsim: $(PRJ)_sim
+	$(MAXCOMPILERSIM) -n $(USER)a -c$(DEVICENUM) -k restart
+	SLIC_CONF+="use_simulation=$(USER)a" LD_PRELOAD=$(MAXELEROSDIR_SIM)/lib/libmaxeleros.so \
+	./$(PRJ)_sim chr1.fmt.trim chr1.trim.fastq
+	$(MAXCOMPILERSIM) -n $(USER)a -c$(DEVICENUM) stop
+
+stopsim:
+	$(MAXCOMPILERSIM) -n $(USER)a -c$(DEVICENUM) stop
+
+# hardware
+$(D1DFEDIR)/$(D1).max: $(D1FILES)
+	$(MAXJC) $(JFLAGS) $(D1FILES)
+	MAXAPPJCP=. MAXSOURCEDIRS='../src' $(MAXJAVARUN) -v -m 8192 $(D1)Manager DFEModel=$(DFEModel) maxFileName=$(D1) target='DFE' enableMPCX=$(MPCX)
+	cp $(D1DFEDIR)/$(D1).h $(D1DFEDIR)/Maxfiles.h
+	rm $(D1DFEDIR)/$(D1).h
+	rm $(D1DFEDIR)/$(D1).max
+	cp $(D1DFEDIR)/../scratch/$(D1).h $(D1DFEDIR)/$(D1).h
+	cp $(D1DFEDIR)/../scratch/altera_quartus/$(D1).max $(D1DFEDIR)/$(D1).max
+	rm -rf $(D1DFEDIR)/../scratch
+	@ mkdir -p class
+	mv *.class class/
+
+$(OBJDIR)/$(D1)_dfe.o: $(D1DFEDIR)/$(D1).max
+	@ mkdir -p obj
+	$(SLICCOMPILE) $< $@
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.cpp $(DEPFILES)
+	@ mkdir -p obj
+	$(CC) $< $(CFLAGS) -I$(D1DFEDIR) -DDESIGN_NAME=$(PRJ) -c -o $@
+
+$(PRJ)_dfe: $(OBJDIR)/$(D1)_dfe.o  $(OBJFILES)
+	$(CC) $(CFLAGS) -o $@ $^ $(LFLAGS)
+
+build: $(PRJ)_dfe
+
+run:
+	SLIC_CONF=$(SLIC_CONF) LD_PRELOAD=/opt/maxeler/maxeleros/lib/libmaxeleros.so \
+	./${PRJ}_dfe /mnt/ssd2/jma11/index/basic/hg38.fmt /mnt/ssd2/jma11/reads/hg38.fastq

applications/fm-index/src/CmdKernel.maxj

@@ -0,0 +1,55 @@
+/* CmdKernel.maxj ----- James Arram 2015 */
+
+import static com.maxeler.maxcompiler.v2.kernelcompiler.types.composite.DFEStructType.sft;
+import com.maxeler.maxcompiler.v2.kernelcompiler.Kernel;
+import com.maxeler.maxcompiler.v2.kernelcompiler.KernelParameters;
+import com.maxeler.maxcompiler.v2.kernelcompiler.stdlib.LMemCommandStream;
+import com.maxeler.maxcompiler.v2.kernelcompiler.types.base.DFEVar;
+import com.maxeler.maxcompiler.v2.kernelcompiler.types.composite.DFEStruct;
+import com.maxeler.maxcompiler.v2.kernelcompiler.types.composite.DFEStructType;
+
+class CmdKernel extends Kernel {
+
+    CmdKernel(KernelParameters parameters) {
+	super(parameters);
+
+	// memory command
+	DFEStructType memCmd_t = new DFEStructType(
+						   sft("lowAddr", dfeUInt(32)),
+						   sft("highAddr", dfeUInt(32)),
+						   sft("highCtrl", dfeBool()),
+						   sft("valid", dfeBool()));
+
+	// kernel input
+	io.pushInputRegistering(false); 
+	DFEVar trigger = io.input("flushTrig", dfeBool());
+	io.popInputRegistering();
+	DFEStruct memCmd = io.input("memCmd", memCmd_t);	
+	flush.onTrigger(trigger);
+
+	// get command values
+	DFEVar lowAddr = memCmd.get("lowAddr");
+	DFEVar highAddr = memCmd.get("highAddr");
+	DFEVar highCtrl = memCmd.get("highCtrl");
+	DFEVar valid = memCmd.get("valid");
+
+	// generate memory command
+	LMemCommandStream.makeKernelOutput("lowCmd",
+					   valid === true,
+					   lowAddr,
+					   constant.var(dfeUInt(8), 1),
+					   constant.var(dfeUInt(1), 0),
+					   constant.var(dfeUInt(1), 0),
+					   constant.var(false));
+
+	// generate memory command
+	LMemCommandStream.makeKernelOutput("highCmd",
+					   valid === true & highCtrl === true,
+					   highAddr,
+					   constant.var(dfeUInt(8), 1),
+					   constant.var(dfeUInt(1), 0),
+					   constant.var(dfeUInt(1), 0),
+					   constant.var(false));
+
+    }
+}

applications/fm-index/src/EmKernel.maxj

@@ -0,0 +1,206 @@
+/* EmKernel.maxj ----- James Arram 2016 */
+
+/*
+ * Notes:
+ * 1) Bucket size is 128 symbols
+ * 2) Number of steps for FM-index is 1
+ * 3) Burst size is 64 bytes
+ */
+
+import static com.maxeler.maxcompiler.v2.kernelcompiler.types.composite.DFEStructType.sft;
+import com.maxeler.maxcompiler.v2.kernelcompiler.Kernel;
+import com.maxeler.maxcompiler.v2.kernelcompiler.KernelParameters;
+import com.maxeler.maxcompiler.v2.kernelcompiler.RoundingMode;
+import com.maxeler.maxcompiler.v2.kernelcompiler.types.base.DFEVar;
+import com.maxeler.maxcompiler.v2.kernelcompiler.types.composite.DFEStruct;
+import com.maxeler.maxcompiler.v2.kernelcompiler.types.composite.DFEStructType;
+import com.maxeler.maxcompiler.v2.kernelcompiler.types.composite.DFEVector;
+import com.maxeler.maxcompiler.v2.kernelcompiler.types.composite.DFEVectorType;
+import com.maxeler.maxcompiler.v2.utils.Bits;
+
+class EmKernel extends Kernel {
+
+    private static final int BUCKET_SIZE = 128;
+
+    private static final int MAX_READ_LENGTH = 168;
+
+    EmKernel(KernelParameters parameters) {
+	super(parameters);
+
+	optimization.pushRoundingMode(RoundingMode.TRUNCATE);
+
+	// read symbols
+	DFEVectorType<DFEVar> sym_t = new DFEVectorType<DFEVar>(dfeUInt(2), MAX_READ_LENGTH);
+
+	// read input
+	DFEStructType in_t = new DFEStructType(
+					       sft("id", dfeUInt(32)),
+					       sft("sym", sym_t),
+					       sft("len", dfeUInt(8)),
+					       sft("isPad", dfeUInt(8)));
+
+	// alignment output
+	DFEStructType out_t = new DFEStructType(
+						sft("id", dfeUInt(32)),
+						sft("low", dfeUInt(32)),
+						sft("high", dfeUInt(32)),
+						sft("pad", dfeUInt(32)));
+
+	// memory command
+	DFEStructType memCmd_t = new DFEStructType(
+						   sft("lowAddr", dfeUInt(32)),
+						   sft("highAddr", dfeUInt(32)),
+						   sft("highCtrl", dfeBool()),
+						   sft("valid", dfeBool()));
+
+
+	// kernel control counters
+	DFEVar offset = io.scalarInput("offset", dfeInt(32));
+	DFEVar tickCount = control.count.simpleCounter(32, offset.cast(dfeUInt(32)));
+	DFEVar tickInit = control.count.pulse(1, tickCount === offset.cast(dfeUInt(32)) - 1);
+
+	// input stream control
+	DFEVar inputCtrlHead = dfeBool().newInstance(this);
+	DFEVar inputCtrl = tickInit === 1 ? constant.var(true) : inputCtrlHead;
+	DFEVar memCtrlHead = dfeBool().newInstance(this);
+	DFEVar memCtrl = inputCtrl === true ? constant.var(false) : memCtrlHead;
+
+	// kernel input
+	DFEStruct readIn = io.input("readIn", in_t, inputCtrl === true);
+	DFEVar lowIndex = io.input("lowIndex", dfeRawBits(512), inputCtrl === false);
+	DFEVar highIndex = io.input("highIndex", dfeRawBits(512), memCtrl === true);
+	DFEVar highInit = io.scalarInput("highInit", dfeUInt(32));	
+	flush.whenInputFinished("readIn");
+
+	// set up streams
+	DFEVar idIn = readIn.get("id");
+	DFEVar idHead = dfeUInt(32).newInstance(this);
+	DFEVar id = inputCtrl === true ? idIn : idHead;	
+
+	DFEVector<DFEVar> symIn = readIn.get("sym");
+	DFEVector<DFEVar> symHead = sym_t.newInstance(this);
+	DFEVector<DFEVar> sym = inputCtrl === true ? symIn : symHead;
+
+	DFEVar lenIn = readIn.get("len");
+	DFEVar lenHead = dfeUInt(8).newInstance(this);
+	DFEVar len = inputCtrl === true ? lenIn : lenHead;
+
+	DFEVar isPadIn = readIn.get("isPad");
+	DFEVar isPadHead = dfeUInt(8).newInstance(this);
+	DFEVar isPad = inputCtrl === true ? isPadIn : isPadHead;
+
+	DFEVar low = dfeUInt(32).newInstance(this);
+	DFEVar high = dfeUInt(32).newInstance(this);
+
+	DFEVar idxHead = dfeUInt(8).newInstance(this);
+	DFEVar idx = inputCtrl === true ? constant.var(dfeUInt(8), 0) : idxHead;
+
+	// get read symbol
+	DFEVar symVal = sym[0];
+
+	// update SA interval
+	DFEVar lowTmp = low === 0 ? low : low - 1;
+	DFEVar lowCounter = getCounter(symVal, lowIndex);
+	DFEVar highCounter = memCtrl === true ? getCounter(symVal, highIndex) : lowCounter;
+	DFEVar lowIdx = lowTmp & 0x7F;
+	DFEVar highIdx = high & 0x7F;
+	DFEVar lowCount = getCount(symVal, lowIndex, lowIdx);
+	DFEVar highCount = memCtrl === true ? getCount(symVal, highIndex, highIdx) :
+	    getCount(symVal, lowIndex, highIdx); 
+	DFEVar lowUp = low === 0 ? lowCounter : 
+	    lowCounter + lowCount.cast(dfeUInt(32));
+	DFEVar highUp = highCounter + highCount.cast(dfeUInt(32)) - 1;
+
+	// update alignment state
+	DFEVar lowNew = inputCtrl === true ? constant.var(dfeUInt(32), 0) : lowUp;
+	DFEVar highNew = inputCtrl === true ? highInit : highUp;
+	DFEVar isAlign = lowNew <= highNew ? constant.var(true) : constant.var(false);
+	DFEVar idxNew = inputCtrl === true ? idx : idx + 1;
+	DFEVar shift = inputCtrl === true ? 0 : constant.var(dfeUInt(8), 1);
+	DFEVector<DFEVar> symNew = sym.shiftElementsRight(shift);
+	DFEVar inputCtrlNew = ((idxNew === len) | (isAlign === false)) & (isPad === 0) ? 
+	    constant.var(true) : constant.var(false);
+
+	// generate new memory command
+	DFEVar lowNewTmp = lowNew === 0 ? lowNew : lowNew - 1;
+	DFEVar lowNewAddr = lowNewTmp>>7;
+	DFEVar highNewAddr = highNew>>7;
+	DFEVar memCtrlNew = (lowNewAddr !== highNewAddr) & inputCtrlNew === false ?
+	    constant.var(true) : constant.var(false);
+	DFEStruct memCmd = memCmd_t.newInstance(this);
+	memCmd.set("lowAddr", lowNewAddr);
+	memCmd.set("highAddr", highNewAddr);
+	memCmd.set("highCtrl", memCtrlNew);
+	memCmd.set("valid", inputCtrlNew === false);
+	io.output("memCmd", memCmd, memCmd_t);
+	io.output("flushTrig", isPad === 2, dfeBool());
+	flush.allowOutputBeforeFlush("memCmd");
+	flush.allowOutputBeforeFlush("flushTrig");
+
+	// connect streams
+	idHead <== stream.offset(id, -offset, -200, -50);
+	symHead <== stream.offset(symNew, -offset, -200, -50);
+	isPadHead <== stream.offset(isPad, -offset, -200, -50);
+	lenHead <== stream.offset(len, -offset, -200, -50);
+	idxHead <== stream.offset(idxNew, -offset, -200, -50);
+	inputCtrlHead <== stream.offset(inputCtrlNew, -offset, -200, -50);
+	memCtrlHead <== stream.offset(memCtrlNew, -offset, -200, -50);
+	low <== stream.offset(lowNew, -offset, -200, -50);
+	high <== stream.offset(highNew, -offset, -200, -50);
+
+	// kernel output
+	DFEStruct out = out_t.newInstance(this);
+	out.set("id", id);
+	out.set("low", lowNew);
+	out.set("high", highNew);
+	out.set("pad", constant.var(dfeUInt(32), 0));
+	io.output("alignOut", out, out_t, inputCtrlNew === true);
+    }
+
+    // get counter
+    private DFEVar getCounter(DFEVar sym, DFEVar x) {
+	DFEVectorType<DFEVar> counters_t = new DFEVectorType<DFEVar>(dfeUInt(32), 4);
+	DFEVector<DFEVar> counters = counters_t.unpack(x.slice(0, 128));
+
+	return control.mux(sym, counters.getElementsAsList());
+    }
+
+    // get count
+    private DFEVar getCount(DFEVar sym, DFEVar x, DFEVar idx) {
+	DFEVectorType<DFEVar> count_t = new DFEVectorType<DFEVar>(dfeUInt(1), BUCKET_SIZE);
+	DFEVector<DFEVar> count = count_t.newInstance(this);
+
+	DFEVectorType<DFEVar> bwt_t = new DFEVectorType<DFEVar>(dfeUInt(3), BUCKET_SIZE);
+	DFEVector<DFEVar> bwt = bwt_t.unpack(x.slice(128, 384));
+
+	for (int i = 0; i < BUCKET_SIZE; i++) {
+	    count[i] <== (i <= idx) & (sym.cast(dfeUInt(3)) === bwt[i]) ? 
+		constant.var(dfeUInt(1), 1) : constant.var(dfeUInt(1), 0);
+	}
+
+	DFEVar[] res = new DFEVar[2];
+	for (int i = 0; i < 2; i++) {
+	    res[i] = popCount(count.pack().slice(i*64, 64).cast(dfeUInt(64)));
+	}
+	return res[0] + res[1];
+    }
+
+    // pop count
+    private DFEVar popCount(DFEVar x) {
+	Bits m1Bits = new Bits(64, "5555555555555555");
+	Bits m2Bits = new Bits(64, "3333333333333333");
+	Bits m4Bits = new Bits(64, "0f0f0f0f0f0f0f0f");
+
+	DFEVar m1 = constant.var(dfeUInt(64), m1Bits);
+	DFEVar m2 = constant.var(dfeUInt(64), m2Bits);
+	DFEVar m4 = constant.var(dfeUInt(64), m4Bits);
+
+	x -= (x >> 1) & m1;
+	x = (x & m2) + ((x >> 2) & m2);
+	x = (x + (x >> 4)) & m4;
+	x += x >>  8;
+	x += x >> 16;
+	x += x >> 32;
+	return x & 0x7f;
+    }
+}