Set up application of thresholds during classification.

fionamac89 · Mar 19, 2014 · 023ce51 · 023ce51
1 parent 23dc6e4
commit 023ce51
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 18 deletions.
diff --git a/src/database/Database.java b/src/database/Database.java
@@ -717,6 +717,40 @@ public void dbCreateEvalTable(String name) {
 		}
 	}
 
+	public List<String> dbApplyThreshold(String name, int genreid, int upper, int lower) {
+		PreparedStatement pst = null;
+		ResultSet rs = null;
+		List<String> words = new ArrayList<String>();
+		String sql = "";
+		try {
+			sql = "SELECT * FROM " + name + " WHERE GenreID=? AND Frequency < ? AND Frequency > ?";
+			pst = con.prepareStatement(sql);
+			pst.setInt(1, genreid);
+			pst.setInt(2, upper);
+			pst.setInt(3, lower);
+			rs = pst.executeQuery();
+			while (rs.next()) {
+				words.add(rs.getString("Word"));
+			}
+		} catch (SQLException e) {
+			e.printStackTrace();
+		} finally {
+			try {
+				if (pst != null) {
+					pst.close();
+				}
+				if (rs != null) {
+					rs.close();
+				}
+
+			} catch (SQLException ex) {
+				Logger lgr = Logger.getLogger(Database.class.getName());
+				lgr.log(Level.WARNING, ex.getMessage(), ex);
+			}
+		}
+		return words;
+	}
+
 	public void dbCreateEvalGenreTable(String name) {
 		PreparedStatement pst = null;
 		try {

diff --git a/src/main/ISystem.java b/src/main/ISystem.java
@@ -22,4 +22,5 @@ public interface ISystem {
 	public void testEval(int tp, int fp, int fn);
 	public void runEvalPerGenre(String evalName, String suffix, String className);
 	public void createEvalGenre(String evalName);
+	public void trainClassifier2(String thesName, int i, int j);
 }
diff --git a/src/main/ProjectSystem.java b/src/main/ProjectSystem.java
@@ -220,6 +220,19 @@ public void classifyTestData(String suffix) {
 
 	}
 
+	public void trainClassifier2(String name, int upper, int lower) {
+		Map<Integer, String> genreList = db.dbGetGenreListMap();
+
+		for (Integer genreid : genreList.keySet()) {
+			List<String> wordsList = db.dbApplyThreshold(name, genreid, upper, lower);
+			cls.addToDataset(genreList.get(genreid), cls.readLines(wordsList));
+		}
+		cls.trainClassifier();
+		cls.setKnowledgeBase();
+		cls.resetClassifier();
+		System.out.println("Classifier Trained");
+	}
+
 	public void createClassified(String name) {
 		db.dbCreateClassifiedTable(name);
 		System.out.println("Classified Table Created");
@@ -263,7 +276,7 @@ public void runEvalPerGenre(String table, String test, String classified) {
 			if (testMap.size() > 0 && classifiedMap.size() > 0) {
 				genre_count++;
 				eval.runEvaluation(testMap, classifiedMap);
-				
+
 				db.dbAddEvalGenre(table, genreid, "Precision",
 						eval.getPrecision());
 				db.dbAddEvalGenre(table, genreid, "Recall", eval.getRecall());
@@ -273,22 +286,22 @@ public void runEvalPerGenre(String table, String test, String classified) {
 				precision += eval.getPrecision();
 				recall += eval.getRecall();
 				fmeasure += eval.getFmeasure();
-				
+
 				System.out.println(genreid);
 				System.out.println("Precision: " + eval.getPrecision());
 				System.out.println("Recall: " + eval.getRecall());
 				System.out.println("Fmeasure: " + eval.getFmeasure());
 			}
 		}
-		
-		precision = (precision/genre_count);
-		recall = (recall/genre_count);
-		fmeasure = (fmeasure/genre_count);
-		
-		db.dbCreateEvalTable("Total_"+table);
-		db.dbAddEval("Total_"+table, classified, "Precision", precision);
-		db.dbAddEval("Total_"+table, classified, "Recall", recall);
-		db.dbAddEval("Total_"+table, classified, "Fmeasure", fmeasure);
+
+		precision = (precision / genre_count);
+		recall = (recall / genre_count);
+		fmeasure = (fmeasure / genre_count);
+
+		db.dbCreateEvalTable("Total_" + table);
+		db.dbAddEval("Total_" + table, classified, "Precision", precision);
+		db.dbAddEval("Total_" + table, classified, "Recall", recall);
+		db.dbAddEval("Total_" + table, classified, "Fmeasure", fmeasure);
 
 	}
 
@@ -312,4 +325,5 @@ public void testEval(int tp, int fp, int fn) {
 		// eval.setFmeasure();
 		System.out.println("Fmeasure: " + eval.getFmeasure());
 	}
+
 }
diff --git a/src/main/Run.java b/src/main/Run.java
@@ -14,9 +14,9 @@ public static void main(String[] args) {
 		//system.addMovieList("/Users/Fiona/Dropbox/Strath Uni/Year 4/Project/Script Test/output3.txt", list, fg);
 		/*Genre File: "/Users/Fiona/Dropbox/Strath Uni/Year 4/Project/Script Test/genres.txt"*/
 		//system.createTrainingSet(250);
-		String thesName = "Thesaurus2_S_250_NewAlg";
-		String className = "NFClassified2_None_250";
-		String suffix = "250_2";
+		String thesName = "Thesaurus2_SSW_1000_NewAlg";
+		String className = "NFClassified2_SSW_1000_NewAlg";
+		String suffix = "1000_2";
 		String evalName = "Eval_None_250";
 		//system.createTrainingTable(suffix);
 		//system.createTestTable(suffix);
@@ -25,12 +25,14 @@ public static void main(String[] args) {
 		//system.createThesaurus(thesName);
 		//system.populateThesaurus2("s", thesName, suffix);
 		//system.createClassified(className);
-		//system.trainClassifier(thesName);
-		//system.classifyTestData(suffix);
-		//system.archiveClassified(className);
+		system.trainClassifier2(thesName, 200, 5);
+		system.classifyTestData(suffix);
+		system.archiveClassified(className);
 
 		//system.createEvalGenre(evalName);
-		system.runEvalPerGenre(evalName, suffix, className);
+		//system.runEvalPerGenre(evalName, suffix, className);
+
+		//system.enforceThresholds("Thesaurus2_SSW_1000_NewAlg", 200, 5);
 
 	}