diff --git a/README.md b/README.md index 9444c18..c33c51c 100644 --- a/README.md +++ b/README.md @@ -66,26 +66,28 @@ Follow these steps to setup XIVStats-Gatherer-Java: ``` The application can be run with the following command line options/args: - | Short option | Long option | Argument type | Description | - |:------------:|:---------------------:|:--------------:|:---------------------------------------------------------:| - |-b |--do-not-store-progress| none | do not store boolean data indicating player progress | - |-d | --database | String | database name | - |-f | --finish | integer | the character id to conclude character run at (inclusive) | - |-F | --print-failures | none | print records that don't exist | - |-h | --help | none | display help message | - |-m | --store-mounts | none | store mount data set for each player into the database | - |-P | --store-minions | none | store minion data set for each player into the database | - |-p | --password | String | database user password | - |-q | --quiet | none | run program in quiet mode - no console output | - |-s | --start | integer | the character id to start from (inclusive) | - |-S | --split-table | none | split table into several small tables | - |-t | --threads | integer | number of gatherer thrads to running | - |-T | --table | String | the table to write records to | - |-u | --user | String | database user | - |-U | --url | String | the database URL of the database server to connect to | - |-v | --verbose | none | run program in verbose mode - full console output | - |-x | --suffix | String | suffix to append to all tables generated | - + | Short option | Long option | Argument type | Description | + |:------------:|:---------------------:|:--------------:|:--------------------------------------------------------------------:| + |-a |--do-not-store-activity| none | do not store boolean data indicating player activity in last 30 days | + |-b |--do-not-store-progress| none | do not store boolean data indicating player progress | + |-d | --database | String | database name | + |-D | --do-not-store-date | none | do not store date of last player activity | + |-f | --finish | integer | the character id to conclude character run at (inclusive) | + |-F | --print-failures | none | print records that don't exist | + |-h | --help | none | display help message | + |-m | --store-mounts | none | store mount data set for each player into the database | + |-P | --store-minions | none | store minion data set for each player into the database | + |-p | --password | String | database user password | + |-q | --quiet | none | run program in quiet mode - no console output | + |-s | --start | integer | the character id to start from (inclusive) | + |-S | --split-table | none | split table into several small tables | + |-t | --threads | integer | number of gatherer thrads to running | + |-T | --table | String | the table to write records to | + |-u | --user | String | database user | + |-U | --url | String | the database URL of the database server to connect to | + |-v | --verbose | none | run program in verbose mode - full console output | + |-x | --suffix | String | suffix to append to all tables generated | + Note: On Linux/Unix it is advised to run the program in Tmux/Screen or similar. @@ -175,6 +177,8 @@ The database table ```tblplayers``` has the following structure: |legacy_player |bit |Mount - Legacy Chocobo | |*mounts* |*text* |*N/A* | |*minions* |*text* |*N/A* | +|date_active |date |N/A | +|is_active |bit |N/A | *Italicised fields are only completed jf specified with a command line flag.* diff --git a/pom.xml b/pom.xml index 57fb5a3..6e03355 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.ffxivcensus.gatherer XIVStats-Gatherer-Java - v1.1.0 + v1.2.0 XIVStats Lodestone Gatherer https://github.com/xivstats @@ -155,6 +155,31 @@ commons-cli 1.3.1 + + org.apache.httpcomponents + httpclient + 4.3.6 + + + org.apache.httpcomponents + httpasyncclient + 4.0.2 + + + org.apache.httpcomponents + httpmime + 4.3.6 + + + org.json + json + 20140107 + + + com.mashape.unirest + unirest-java + 1.4.9 + diff --git a/src/main/java/com/ffxivcensus/gatherer/Console.java b/src/main/java/com/ffxivcensus/gatherer/Console.java index 3092f15..88864c9 100644 --- a/src/main/java/com/ffxivcensus/gatherer/Console.java +++ b/src/main/java/com/ffxivcensus/gatherer/Console.java @@ -26,7 +26,7 @@ public static GathererController run(String [] args){ Options options = setupOptions(); //Declare usage string - String usage = "java -jar XIVStats-Gatherer-Java.jar [-bmqvxFPS] -s startid -f finishid [-d database-name] [-u database-user] [-p database-user-password] [-U database-url] [-T table] [-t threads]"; + String usage = "java -jar XIVStats-Gatherer-Java.jar [-abmqvxDFPS] -s startid -f finishid [-d database-name] [-u database-user] [-p database-user-password] [-U database-url] [-T table] [-t threads]"; HelpFormatter formatter = new HelpFormatter(); try{ @@ -62,6 +62,12 @@ public static GathererController run(String [] args){ //Store progression gatherer.setStoreProgression(!cmd.hasOption("b")); + //Store whether player is active + gatherer.setStorePlayerActive(!cmd.hasOption("a")); + + //Store player active date + gatherer.setStoreActiveDate(!cmd.hasOption("D")); + //Database URL if(cmd.hasOption("d") && cmd.hasOption("U")){ gatherer.setDbUrl("jdbc:" + cmd.getOptionValue("U") + "/" + cmd.getOptionValue("d")); @@ -143,6 +149,8 @@ public static Options setupOptions(){ Option optVerbose = Option.builder("v").longOpt("verbose").desc("run program in verbose bug mode - full console output").build(); Option optFailPrint = Option.builder("F").longOpt("print-failures").desc("print records that don't exist").build(); Option optSuffix = Option.builder("x").longOpt("suffix").hasArg().numberOfArgs(1).argName("table-suffix").desc("suffix to append to all tables").build(); + Option optStoreActive = Option.builder("a").longOpt("do-not-store-activity").desc("do not store boolean data indicating player activity in last 30 days").build(); + Option optStoreDate = Option.builder("D").longOpt("do-not-store-date").desc("do not store Date of last player activity").build(); //Add each option to the options object options.addOption(optStart); @@ -162,6 +170,8 @@ public static Options setupOptions(){ options.addOption(optVerbose); options.addOption(optFailPrint); options.addOption(optSuffix); + options.addOption(optStoreActive); + options.addOption(optStoreDate); return options; } diff --git a/src/main/java/com/ffxivcensus/gatherer/GathererController.java b/src/main/java/com/ffxivcensus/gatherer/GathererController.java index eef3629..c8c8a96 100644 --- a/src/main/java/com/ffxivcensus/gatherer/GathererController.java +++ b/src/main/java/com/ffxivcensus/gatherer/GathererController.java @@ -92,6 +92,14 @@ public class GathererController { * Whether to output failed records */ private boolean printFails; + /** + * Whether to store player activity dates + */ + private boolean storeActiveDate; + /** + * Whether to store player activity bit + */ + private boolean storePlayerActive; /** * List of playable realms (used when splitting tables). @@ -228,6 +236,8 @@ public GathererController(int startId, int endId, boolean quiet, boolean verbose this.tableName = "tblplayers"; this.tableSuffix = tableSuffix; this.splitTables = splitTables; + this.storeActiveDate = true; + this.storePlayerActive = true; } /** @@ -344,6 +354,14 @@ private void createTable(String tableName) { if (this.storeMinions) { sbSQL.append(",minions TEXT"); } + if(this.storeActiveDate) { + sbSQL.append(","); + sbSQL.append("date_active DATE"); + } + if(this.storePlayerActive) { + sbSQL.append(","); + sbSQL.append("is_active BIT"); + } sbSQL.append(");"); st.executeUpdate(sbSQL.toString()); @@ -509,6 +527,25 @@ protected String writeToDB(Player player) { sbValues.append("\"" + player.getMountsString() + "\""); } + + if(this.storeActiveDate) { + sbFields.append(","); + sbValues.append(","); + sbFields.append("date_active"); + java.util.Date dt = new java.util.Date(); + + java.text.SimpleDateFormat sdf = new java.text.SimpleDateFormat("yyyy-MM-dd"); + + String sqlDate = sdf.format(player.getDateImgLastModified()); + sbValues.append("\"" + sqlDate + "\""); + } + if(this.storePlayerActive) { + sbFields.append(","); + sbValues.append(","); + sbFields.append("is_active"); + sbValues.append(player.getBitIsActive()); + } + sbFields.append(")"); sbValues.append(");"); @@ -734,7 +771,7 @@ public void setVerbose(boolean verbose) { } /** * Get list of realms to create tables for - * @return + * @return array of realm names */ public static String[] getRealms() { return realms; @@ -787,4 +824,20 @@ public boolean isPrintFails() { public void setPrintFails(boolean printFails) { this.printFails = printFails; } + + /** + * Set whether to store the last active date of a character + * @param storeActiveDate whether to store the last active date of a character + */ + public void setStoreActiveDate(boolean storeActiveDate) { + this.storeActiveDate = storeActiveDate; + } + + /** + * Set whether to store a boolean value indicating player activity + * @param storePlayerActive whether to store a boolean value indicating player activity + */ + public void setStorePlayerActive(boolean storePlayerActive) { + this.storePlayerActive = storePlayerActive; + } } diff --git a/src/main/java/com/ffxivcensus/gatherer/Player.java b/src/main/java/com/ffxivcensus/gatherer/Player.java index a27bf39..f7dc912 100644 --- a/src/main/java/com/ffxivcensus/gatherer/Player.java +++ b/src/main/java/com/ffxivcensus/gatherer/Player.java @@ -1,15 +1,21 @@ package com.ffxivcensus.gatherer; +import com.mashape.unirest.http.HttpResponse; +import com.mashape.unirest.http.JsonNode; +import com.mashape.unirest.http.Unirest; +import com.mashape.unirest.http.exceptions.UnirestException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; import java.util.regex.Pattern; /** @@ -22,6 +28,20 @@ */ public class Player { + /** + * Number of days inactivity before character is considered inactive + */ + private final static int ACTIVITY_RANGE_DAYS = 30; + + + private static final long ONE_MINUTE_IN_MILLIS=60000; + private static final long ONE_DAY_IN_MILLIS=86400000; + + /** + * Ignore dates from inside EXCLUDE_RANGE in minutes + */ + private static final long EXCLUDE_RANGE= 5; + private int id; private String realm; private String playerName; @@ -86,6 +106,8 @@ public class Player { private boolean isLegacyPlayer; private ArrayList minions; private ArrayList mounts; + private Date dateImgLastModified; + private boolean isActive; /** * Constructor for player object. @@ -154,6 +176,8 @@ public Player(int id) { setHasCompletedHW(false); setHasCompleted3pt1(false); setHasCompleted3pt3(false); + setDateImgLastModified(new Date()); + setActive(false); } /** @@ -1654,8 +1678,16 @@ public int getBitHasCompleted3pt3() { */ public void setHasCompleted3pt3(boolean hasCompleted3pt3) { this.hasCompleted3pt3 = hasCompleted3pt3; - } - + } + + /** + * Set the date on which the player's avatar was last modified + * @param dateImgLastModified the date on which the player's avatar was last modified + */ + public void setDateImgLastModified(Date dateImgLastModified) { + this.dateImgLastModified = dateImgLastModified; + } + /** * Get whether the user played 1.0. * @@ -1830,6 +1862,7 @@ public static Player getPlayer(int playerID) throws Exception { player.setGender(getGenderFromPage(doc)); player.setGrandCompany(getGrandCompanyFromPage(doc)); player.setFreeCompany(getFreeCompanyFromPage(doc)); + player.setDateImgLastModified(getDateLastUpdatedFromPage(doc)); player.setLevels(getLevelsFromPage(doc)); player.setMounts(getMountsFromPage(doc)); player.setMinions(getMinionsFromPage(doc)); @@ -1865,12 +1898,31 @@ public static Player getPlayer(int playerID) throws Exception { player.setHasSylph(player.doesPlayerHaveMount("Laurel Goobbue")); player.setHasCompletedHW(player.doesPlayerHaveMount("Midgardsormr")); player.setIsLegacyPlayer(player.doesPlayerHaveMount("Legacy Chocobo")); + player.setActive(player.isPlayerActiveInDateRange()); } catch (IOException ioEx) { throw new Exception("Character " + playerID + " does not exist."); } return player; } + /** + * Determine whether a player is active based upon the last modified date of their full body image + * @return whether player has been active inside the activity window + */ + private boolean isPlayerActiveInDateRange() { + + Calendar date = Calendar.getInstance(); + long t= date.getTimeInMillis(); + Date nowMinusExcludeRange =new Date(t - (EXCLUDE_RANGE * ONE_MINUTE_IN_MILLIS)); + + Date nowMinusIncludeRange = new Date(t - (ACTIVITY_RANGE_DAYS * ONE_DAY_IN_MILLIS)); + if(this.dateImgLastModified.after(nowMinusExcludeRange)) { //If the date modified is inside the exclude range + //Reset the last modified date to epoch because we aren't considering it valid + this.dateImgLastModified = new Date(0); + return false; + } else return this.dateImgLastModified.after(nowMinusIncludeRange); //If the date occurs between the include range and now, then return true. Else false + } + /** * Given a lodestone profile page, return the name of the character. * @@ -2077,4 +2129,67 @@ private static ArrayList getMountsFromPage(Document doc) { return mounts; } + /** + * Gets the last-modified date of the Character full body image. + * @param doc the lodestone profile page to parse + * @return the date on which the full body image was last modified. + */ + private static Date getDateLastUpdatedFromPage(Document doc) throws Exception { + Date dateLastModified = new Date(); + //Get character image URL. + String imgUrl = doc.getElementsByClass("bg_chara_264").get(0).getElementsByTag("img").get(0).attr("src"); + String strLastModifiedDate = ""; + + try { + HttpResponse jsonResponse = Unirest.head(imgUrl).asJson(); + + strLastModifiedDate = jsonResponse.getHeaders().get("Last-Modified").toString(); + } catch (UnirestException e) { + e.printStackTrace(); + } + + strLastModifiedDate = strLastModifiedDate.replace("[", ""); + strLastModifiedDate = strLastModifiedDate.replace("]", ""); + DateFormat dateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); + + try { + dateLastModified = dateFormat.parse(strLastModifiedDate); + } catch (ParseException e) { + throw new Exception("Could not correctly parse date 'Last-Modified' header from full body image"); + } + return dateLastModified; + } + + /** + * Get the date on which the Character's full body image was last modified + * @return the date on which the Character's full body image was last modified + */ + public Date getDateImgLastModified() { + return dateImgLastModified; + } + + /** + * Get whether a Player is active + * @return whether Player is active + */ + public boolean isActive() { + return isActive; + } + + /** + * Get whether a Player is active + * @return whether Player is active + */ + public int getBitIsActive() { + if(this.isActive) return 1; + return 0; + } + + /** + * Set whether Player is active + * @param active whether player is considered active + */ + public void setActive(boolean active) { + isActive = active; + } } diff --git a/src/test/java/com/ffxivcensus/gatherer/ConsoleTest.java b/src/test/java/com/ffxivcensus/gatherer/ConsoleTest.java index 6a5de7c..5765c3a 100644 --- a/src/test/java/com/ffxivcensus/gatherer/ConsoleTest.java +++ b/src/test/java/com/ffxivcensus/gatherer/ConsoleTest.java @@ -168,7 +168,7 @@ public void testConsoleFullOptions() throws Exception { @Test public void TestConsoleHelpDefault() throws Exception { - String strHelp = "usage: java -jar XIVStats-Gatherer-Java.jar [-bmqvxFPS] -s startid -f"; + String strHelp = "usage: java -jar XIVStats-Gatherer-Java.jar [-abmqvxDFPS] -s startid -f"; //Test for a help dialog displayed upon failure String[] args = {""}; @@ -180,7 +180,7 @@ public void TestConsoleHelpDefault() throws Exception { @Test public void TestConsoleHelpOnFail() throws Exception { - String strHelp = "usage: java -jar XIVStats-Gatherer-Java.jar [-bmqvxFPS] -s startid -f"; + String strHelp = "usage: java -jar XIVStats-Gatherer-Java.jar [-abmqvxDFPS] -s startid -f"; //Test for a help dialog displayed upon failure String[] args = {"-s 0"}; GathererController gc = Console.run(args); @@ -192,7 +192,7 @@ public void TestConsoleHelpOnFail() throws Exception { @Test public void TestConsoleHelp() throws Exception { - String strHelp = "usage: java -jar XIVStats-Gatherer-Java.jar [-bmqvxFPS] -s startid -f"; + String strHelp = "usage: java -jar XIVStats-Gatherer-Java.jar [-abmqvxDFPS] -s startid -f"; //First test for a user requested help dialog String[] args = {"--help"}; diff --git a/src/test/java/com/ffxivcensus/gatherer/PlayerTest.java b/src/test/java/com/ffxivcensus/gatherer/PlayerTest.java index e2fd961..b197c78 100644 --- a/src/test/java/com/ffxivcensus/gatherer/PlayerTest.java +++ b/src/test/java/com/ffxivcensus/gatherer/PlayerTest.java @@ -2,6 +2,8 @@ import com.ffxivcensus.gatherer.Player; +import java.util.Date; + import static org.junit.Assert.*; /** @@ -154,6 +156,9 @@ public void testGetPlayer() throws Exception { assertTrue(playerOne.getMountsString().contains("Cavalry Drake,Cavalry Elbst")); //Test for data from very end assertTrue(playerOne.getMountsString().contains("Midgardsormr")); + + //Is active + assertTrue(playerOne.isActive()); } /** @@ -221,6 +226,9 @@ public void testUnplayedPlayer() throws Exception { assertEquals(player.getBitHasCompletedHW(), 0); assertEquals(player.getBitHasCompleted3pt1(), 0); assertEquals(player.getBitHasARRCollectors(), 0); + //Tricky to test this - testing here that it was at the very least set to some value other than what it is set to a value other than that which it is initialized + assertTrue(player.getDateImgLastModified() != new Date()); + assertFalse(player.isActive()); //Test get minions method assertTrue(player.getMinions().size() == 0);