Fixed several bugs related to dates and query parameters

run.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 DEFAULT_HADOOP_HOME=/home/user/hadoop-2.6.0 #change to your hadoop folder
-DEFAULT_LDBC_SNB_DATAGEN_HOME=/home/user/ldbc_snb_datagen_0.2 #change to your ldbc_socialnet_dbgen folder
-PARAM_GENERATION=1 #param generation
+DEFAULT_LDBC_SNB_DATAGEN_HOME=/home/user/ldbc_snb_datagen #change to your ldbc_socialnet_dbgen folder
 
 # allow overriding configuration from outside via environment variables
 # i.e. you can do
@@ -14,22 +13,13 @@ export HADOOP_HOME
 export LDBC_SNB_DATAGEN_HOME
 
 mvn clean
-mvn -DskipTests assembly:assembly 
+mvn -DskipTests package 
 
-cp $LDBC_SNB_DATAGEN_HOME/target/ldbc_snb_datagen.jar $LDBC_SNB_DATAGEN_HOME/
-rm $LDBC_SNB_DATAGEN_HOME/target/ldbc_snb_datagen.jar
+$HADOOP_HOME/bin/hadoop jar $LDBC_SNB_DATAGEN_HOME/target/ldbc_snb_datagen-0.2.5.jar ldbc.snb.datagen.generator.LDBCDatagen $LDBC_SNB_DATAGEN_HOME/params.ini
 
-$HADOOP_HOME/bin/hadoop jar $LDBC_SNB_DATAGEN_HOME/ldbc_snb_datagen.jar $LDBC_SNB_DATAGEN_HOME/params.ini
-
-if [ $PARAM_GENERATION -eq 1 ]
-then
-	mkdir -p substitution_parameters
-	python paramgenerator/generateparams.py $LDBC_SNB_DATAGEN_HOME substitution_parameters/
-	python paramgenerator/generateparamsbi.py $LDBC_SNB_DATAGEN_HOME substitution_parameters/
-  rm -f m*personFactors*
-  rm -f .m*personFactors*
-  rm -f m*activityFactors*
-  rm -f .m*activityFactors*
-  rm -f m0friendList*
-  rm -f .m0friendList*
-fi
+rm -f m*personFactors*
+rm -f .m*personFactors*
+rm -f m*activityFactors*
+rm -f .m*activityFactors*
+rm -f m0friendList*
+rm -f .m0friendList*

src/main/java/ldbc/snb/datagen/dictionary/FlashmobTagDictionary.java

@@ -117,7 +117,7 @@ private void initialize() {
                 flashmobTags.put(tags[i], instances);
             }
             FlashmobTag flashmobTag = new FlashmobTag();
-            flashmobTag.date = dateGen.randomDateInMillis(random, dateGen.getStartDateTime(), dateGen.getEndDateTime());
+            flashmobTag.date = dateGen.randomDate(random, dateGen.getStartDateTime());
             flashmobTag.level = levelGenerator.getValue(random);
             sumLevels += flashmobTag.level;
             flashmobTag.tag = tags[i];

src/main/java/ldbc/snb/datagen/generator/CommentGenerator.java

@@ -82,7 +82,7 @@ public long createComments(RandomGeneratorFarm randomFarm, final Forum forum, fi
 			}
 
 			long creationDate = Dictionaries.dates.powerlawCommDateDay(randomFarm.get(RandomGeneratorFarm.Aspect.DATE),replyTo.creationDate()+DatagenParams.deltaTime);
-			if( creationDate <= Dictionaries.dates.getEndDateTime() ) {
+			/*if( creationDate <= Dictionaries.dates.getEndDateTime() )*/ {
 				Comment comment = new Comment(SN.formId(SN.composeId(nextId++,creationDate)),
 					creationDate,
 					member.person(),

src/main/java/ldbc/snb/datagen/generator/DateGenerator.java

@@ -79,7 +79,8 @@ public DateGenerator(Configuration conf, GregorianCalendar from, GregorianCalend
 		toBirthDay_ = tobirthCalendar.getTimeInMillis();
 		calendar_ = new GregorianCalendar();
 		calendar_.setTimeZone(TimeZone.getTimeZone("GMT"));
-		updateThreshold_ = getMaxDateTime() - (long)((getMaxDateTime() - getStartDateTime())*(DatagenParams.updatePortion));
+		//updateThreshold_ = getMaxDateTime() - (long)((getMaxDateTime() - getStartDateTime())*(DatagenParams.updatePortion));
+        updateThreshold_ = getEndDateTime() - (long)((getEndDateTime() - getStartDateTime())*(DatagenParams.updatePortion));
 
 		try {
 			dateFormatter_ = (DateFormatter) Class.forName(conf.get("ldbc.snb.datagen.serializer.dateFormatter")).newInstance();
@@ -93,7 +94,7 @@ public DateGenerator(Configuration conf, GregorianCalendar from, GregorianCalend
 	/*
 	 * Date between from and to
 	 */
-	public Long randomDateInMillis(Random random) {
+	public Long randomPersonCreationDate(Random random) {
 		long date = (long) (random.nextDouble() * (to_ - from_) + from_);
 		calendar_.setTime(new Date(date));
 		return calendar_.getTimeInMillis();
@@ -136,71 +137,67 @@ public static boolean isTravelSeason(long date) {
 		}
 	}
 
-	/*
-	 * Format date in xsd:dateTime format
-	 */
-/*	public String formatDateTime(Long date) {
-		calendar.setTimeInMillis(date);
-		String dateString = formatDate(calendar);
-		return dateString + "T00:00:00";
-	}
-
-	public String formatDateTime(GregorianCalendar date) {
-		String dateString = formatDate(date);
-		return dateString + "T00:00:00";
-	}
-	*/
-
 	public int getNumberOfMonths(long date, int startMonth, int startYear) {
         calendar_.setTimeInMillis(date);
 		int month = calendar_.get(Calendar.MONTH) + 1;
 		int year = calendar_.get(Calendar.YEAR);
-
 		return (year - startYear) * 12 + month - startMonth;
 	}
 
-	public Long randomDateInMillis(Random random, Long from, Long to) {
-		long date = (long) (random.nextDouble() * (to - from) + from);
-		calendar_.setTime(new Date(date));
-
-		return calendar_.getTimeInMillis();
-	}
-
-	public Long randomThirtyDaysSpan(Random random, Long from) {
-		long randomSpanMilis = (long) (random.nextDouble() * (THIRTY_DAYS));
-		return (from + randomSpanMilis);
-	}
-
 	public long randomKnowsCreationDate(Random random, Person personA, Person personB) {
-		long fromDate = Math.max(personA.creationDate(), personB.creationDate());
-		long randomSpanMilis = (long) (random.nextDouble() * (THIRTY_DAYS));
-		return Math.min(fromDate + randomSpanMilis, getEndDateTime());
+		long fromDate = Math.max(personA.creationDate(), personB.creationDate()) + DatagenParams.deltaTime;
+		//long randomSpanMilis = (long) (random.nextDouble() * (THIRTY_DAYS));
+		//return Math.min(fromDate + randomSpanMilis, getEndDateTime() + DatagenParams.deltaTime);
+        return randomDate(random, fromDate, fromDate + THIRTY_DAYS);
 	}
 
 	public long numberOfMonths(Person user) {
 		return numberOfMonths(user.creationDate());
 	}
 
 	public long numberOfMonths(long fromDate) {
-		return (to_ - fromDate) / THIRTY_DAYS;
+		return (to_  - fromDate) / THIRTY_DAYS;
 	}
 
-	public long randomDate(Random random, long minDate) {
-		return (long) (random.nextDouble() * (to_ - minDate) + minDate);
+	/*public long randomDate(Random random, long minDate) {
+		return (long) (random.nextDouble() * (to_+ DatagenParams.deltaTime - minDate) + minDate);
 	}
 
-	public long randomSevenDays(Random random) {
-		return (long) (random.nextDouble() * DateGenerator.SEVEN_DAYS);
-	}
+    public long randomDate(Random random, long minDate, long maxDate) {
+        long to = Math.min(maxDate, to_ + DatagenParams.deltaTime);
+        return (long) (random.nextDouble() * (to - minDate) + minDate);
+    }
 
-	public long powerlawCommDateDay(Random random, long lastCommentCreatedDate) {
-		return (long) (powerDist_.getDouble(random) * ONE_DAY + lastCommentCreatedDate);
-	}
+    public long powerlawCommDateDay(Random random, long lastCommentCreatedDate) {
+        long date = (long) (powerDist_.getDouble(random) * ONE_DAY + lastCommentCreatedDate);
+        return Math.min(to_+DatagenParams.deltaTime,date);
+    }*/
+
+
+    public long randomDate(Random random, long minDate) {
+        long to = Math.max(minDate+THIRTY_DAYS, to_);
+        return (long) (random.nextDouble() * (to - minDate) + minDate);
+    }
+
+    public long randomDate(Random random, long minDate, long maxDate) {
+        long to = maxDate;
+        return (long) (random.nextDouble() * (to - minDate) + minDate);
+    }
+
+    public long powerlawCommDateDay(Random random, long lastCommentCreatedDate) {
+        long date = (long) (powerDist_.getDouble(random) * ONE_DAY + lastCommentCreatedDate);
+        return date;
+    }
+
+
+
+    public long randomSevenDays(Random random) {
+        return (long) (random.nextDouble() * DateGenerator.SEVEN_DAYS);
+    }
 
 	// The birthday is fixed during 1980 --> 1990
 	public long getBirthDay(Random random, long userCreatedDate) {
         calendar_.setTimeInMillis(((long)(random.nextDouble() * (toBirthDay_ - fromBirthDay_)) + fromBirthDay_));
-//        GregorianCalendar aux_calendar = new GregorianCalendar(TimeZone.getTimeZone("GMT"));
         GregorianCalendar  aux_calendar = new GregorianCalendar(calendar_.get(Calendar.YEAR),calendar_.get(Calendar.MONTH), calendar_.get(Calendar.DAY_OF_MONTH),0,0,0);
         aux_calendar.setTimeZone(TimeZone.getTimeZone("GMT"));
 		return aux_calendar.getTimeInMillis();
@@ -247,9 +244,10 @@ public long getEndDateTime() {
 		return to_;
 	}
 
-	public long getMaxDateTime() {
+/*	public long getMaxDateTime() {
 		return to_ + SEVEN_DAYS + deltaTime_;
 	}
+	*/
 
 	public long getUpdateThreshold() {
 		return updateThreshold_;

src/main/java/ldbc/snb/datagen/generator/DistanceKnowsGenerator.java

@@ -46,17 +46,4 @@ boolean know( Person personA, Person personB, int dist, ArrayList<Float> percent
         return false;
     }
 
-    void createKnow( Person personA, Person personB ) {
-        long  creationDate = Dictionaries.dates.randomKnowsCreationDate(
-                randomFarm.get(RandomGeneratorFarm.Aspect.DATE),
-                personA,
-                personB);
-        creationDate = creationDate - personA.creationDate() >= DatagenParams.deltaTime ? creationDate : creationDate + (DatagenParams.deltaTime - (creationDate - personA.creationDate()));
-        creationDate = creationDate - personB.creationDate() >= DatagenParams.deltaTime ? creationDate : creationDate + (DatagenParams.deltaTime - (creationDate - personB.creationDate()));
-        if( creationDate <= Dictionaries.dates.getEndDateTime() ) {
-            float similarity = Person.personSimilarity.Similarity(personA,personB);
-            personB.knows().add(new Knows(personA, creationDate, similarity));
-            personA.knows().add(new Knows(personB, creationDate, similarity));
-        }
-    }
 }

src/main/java/ldbc/snb/datagen/generator/FlashmobPostGenerator.java

@@ -129,7 +129,7 @@ protected PostGenerator.PostInfo generatePostInfo( Random randomTag, Random rand
 		}
 	    double prob = dateDistribution_.nextDouble(randomDate);
 	    postInfo.date = flashmobTag.date - flashmobSpan_/2 + (long)(prob * flashmobSpan_);
-	    if( postInfo.date > Dictionaries.dates.getEndDateTime() ) return null;
+	    //if( postInfo.date > Dictionaries.dates.getEndDateTime() ) return null;
 	    return postInfo;
     }
 }

src/main/java/ldbc/snb/datagen/generator/ForumGenerator.java

@@ -38,14 +38,16 @@ public Forum createWall(RandomGeneratorFarm randomFarm, long forumId, Person per
 
 		TreeSet<Knows> knows = person.knows();
 		for ( Knows k : knows ) {
-			forum.addMember(new ForumMembership(forum.id(), k.creationDate()+DatagenParams.deltaTime, k.to()));
+			long date = Math.max(k.creationDate(), forum.creationDate()+DatagenParams.deltaTime);
+			assert (forum.creationDate() + DatagenParams.deltaTime) <= date : "Forum creation date is larger than knows in wall "+forum.creationDate()+ " " +k.creationDate();
+			forum.addMember(new ForumMembership(forum.id(), date, k.to()));
 		}
 		return forum;
 	}
 
 	public Forum createGroup(RandomGeneratorFarm randomFarm, long forumId, Person person, ArrayList<Person> persons){
 		long date = Dictionaries.dates.randomDate(randomFarm.get(RandomGeneratorFarm.Aspect.DATE), person.creationDate() + DatagenParams.deltaTime);
-		if( date > Dictionaries.dates.getEndDateTime() )  return null;
+		//if( date > Dictionaries.dates.getEndDateTime() )  return null;
 
 		int language = randomFarm.get(RandomGeneratorFarm.Aspect.LANGUAGE).nextInt(person.languages().size());
 		Iterator<Integer> iter = person.interests().iterator();
@@ -82,21 +84,23 @@ public Forum createGroup(RandomGeneratorFarm randomFarm, long forumId, Person pe
 				if (!added.contains(k.to().accountId())) {
 					Random random = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX);
 					date = Dictionaries.dates.randomDate(random,Math.max(forum.creationDate(), k.creationDate()+DatagenParams.deltaTime));
-					if( date < Dictionaries.dates.getEndDateTime() ) {
+					assert forum.creationDate() +DatagenParams.deltaTime <= date : "Forum creation date larger than membership date for knows based members";
+					/*if( date < Dictionaries.dates.getEndDateTime() )*/ {
 						forum.addMember(new ForumMembership(forum.id(), date, k.to()));
 						added.add(k.to().accountId());
 					}
 				}
 			} else { 
-				int friendIdx = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX).nextInt(persons.size());
-				Person member = persons.get(friendIdx);
+				int candidateIndex = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX).nextInt(persons.size());
+				Person member = persons.get(candidateIndex);
 				prob = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP).nextDouble();
 				if (prob < 0.1) {
 					if (!added.contains(member.accountId())) {
 						added.add(member.accountId());
 						Random random = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX);
 						date = Dictionaries.dates.randomDate(random,Math.max(forum.creationDate(), member.creationDate()+DatagenParams.deltaTime));
-						if( date < Dictionaries.dates.getEndDateTime() ) {
+						/*if( date < Dictionaries.dates.getEndDateTime() )*/ {
+							assert forum.creationDate() +DatagenParams.deltaTime <= date : "Forum creation date larger than membership date for block based members";
 							forum.addMember(new ForumMembership(forum.id(), date, new Person.PersonSummary(member)));
 							added.add(member.accountId());
 						}
@@ -110,7 +114,7 @@ public Forum createGroup(RandomGeneratorFarm randomFarm, long forumId, Person pe
 
 	public Forum createAlbum(RandomGeneratorFarm randomFarm, long forumId, Person person,  int numAlbum) {
 		long date = Dictionaries.dates.randomDate(randomFarm.get(RandomGeneratorFarm.Aspect.DATE), person.creationDate() + DatagenParams.deltaTime);
-		if( date > Dictionaries.dates.getEndDateTime() )  return null;
+		//if( date > Dictionaries.dates.getEndDateTime() )  return null;
 		int language = randomFarm.get(RandomGeneratorFarm.Aspect.LANGUAGE).nextInt(person.languages().size());
 		Forum forum = new Forum(SN.formId(SN.composeId(forumId,date)),
 				date, 
@@ -140,7 +144,7 @@ public Forum createAlbum(RandomGeneratorFarm randomFarm, long forumId, Person pe
 			if (prob < 0.7) {
 				Random random = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX);
 				date = Dictionaries.dates.randomDate(random,Math.max(forum.creationDate(), k.creationDate()+DatagenParams.deltaTime));
-				if( date < Dictionaries.dates.getEndDateTime() ) {
+				/*if( date < Dictionaries.dates.getEndDateTime() )*/ {
 					forum.addMember(new ForumMembership(forum.id(), date, k.to()));
 				}
 			}

src/main/java/ldbc/snb/datagen/generator/LDBCDatagen.java

@@ -362,6 +362,11 @@ public static void main(String[] args) /*throws Exception*/ {
         LDBCDatagen datagen = new LDBCDatagen();
         LDBCDatagen.init(conf);
             datagen.runGenerateJob(conf);
+        }catch(AssertionError e ) {
+            System.err.println("Error during execution");
+            System.err.println(e.getMessage());
+            e.printStackTrace();
+            System.exit(1);
         }catch(Exception e ) {
             System.err.println("Error during execution");
             System.err.println(e.getMessage());

src/main/java/ldbc/snb/datagen/generator/LikeGenerator.java

@@ -43,8 +43,10 @@ public void generateLikes(Random random, final Forum forum, final Message messag
 		for (int i = 0; i < numLikes; i++) {
 			ForumMembership membership = memberships.get(startIndex+i);
 			long minDate = message.creationDate() > memberships.get(startIndex+i).creationDate() ? message.creationDate() : membership.creationDate();
-			long date = Math.max(Dictionaries.dates.randomSevenDays(random),DatagenParams.deltaTime) + minDate;
-			if( date <= Dictionaries.dates.getEndDateTime() ) {
+			//long date = Math.max(Dictionaries.dates.randomSevenDays(random),DatagenParams.deltaTime) + minDate;
+			long date = Dictionaries.dates.randomDate(random, minDate, Dictionaries.dates.randomSevenDays(random) + minDate);
+			/*if( date <= Dictionaries.dates.getEndDateTime() )*/ {
+				assert((membership.person().creationDate() + DatagenParams.deltaTime) < date);
 				like.user = membership.person().accountId();
 				like.userCreationDate = membership.person().creationDate();
 				like.messageId = message.messageId();

src/main/java/ldbc/snb/datagen/generator/PersonActivityGenerator.java

@@ -48,11 +48,16 @@ public PersonActivityGenerator( PersonActivitySerializer serializer, UpdateEvent
 	}
 
 	private void generateActivity( Person person, ArrayList<Person> block ) {
-		generateWall(person, block);
-		generateGroups(person, block);
-		generateAlbums(person, block);
-        if(person.creationDate() < Dictionaries.dates.getUpdateThreshold() || !DatagenParams.updateStreams ) {
+        try {
             factorTable_.extractFactors(person);
+            generateWall(person, block);
+            generateGroups(person, block);
+            generateAlbums(person, block);
+        } catch (AssertionError e) {
+            System.out.println("Assertion error when generating activity!");
+            System.out.println(e.getMessage());
+            e.printStackTrace();
+            System.exit(1);
         }
 	}
 

src/main/java/ldbc/snb/datagen/generator/PersonGenerator.java

@@ -42,7 +42,8 @@ public PersonGenerator( Configuration conf, String degreeDistribution ) {
 
     private long composeUserId(long id, long date) {
         long idMask = ~(0xFFFFFFFFFFFFFFFFL << 40);
-        long bucket = (long) (256 * (date - Dictionaries.dates.getStartDateTime()) / (double) Dictionaries.dates.getMaxDateTime());
+        //long bucket = (long) (256 * (date - Dictionaries.dates.getStartDateTime()) / (double) Dictionaries.dates.getMaxDateTime());
+        long bucket = (long) (256 * (date - Dictionaries.dates.getStartDateTime()) / (double) Dictionaries.dates.getEndDateTime());
         return (bucket << 40) | ((id & idMask));
     }
 
@@ -60,7 +61,7 @@ private boolean isUserALargePoster(Person user) {
 
     private Person generateUser() {
 
-        long creationDate = Dictionaries.dates.randomDateInMillis(randomFarm.get(RandomGeneratorFarm.Aspect.DATE));
+        long creationDate = Dictionaries.dates.randomPersonCreationDate(randomFarm.get(RandomGeneratorFarm.Aspect.DATE));
         int countryId = Dictionaries.places.getCountryForUser(randomFarm.get(RandomGeneratorFarm.Aspect.COUNTRY));
         Person person = new Person();
         person.creationDate(creationDate);

src/main/java/ldbc/snb/datagen/generator/PhotoGenerator.java

@@ -77,7 +77,7 @@ public long createPhotos(RandomGeneratorFarm randomFarm, final Forum album, fina
 			}
 			TreeSet<Integer> tags = new TreeSet<Integer>();
 			long date = album.creationDate()+DatagenParams.deltaTime+1000*(i+1);
-			if( date <= Dictionaries.dates.getEndDateTime() ) {
+			/*if( date <= Dictionaries.dates.getEndDateTime() )*/ {
 				long id = SN.formId(SN.composeId(nextId++,date));
 				photo_.initialize(id,date,album.moderator(), album.id(), "photo"+id+".jpg",tags,album.moderator().ipAddress(),album.moderator().browserId(),latt,longt);
 				exporter.export(photo_);

src/main/java/ldbc/snb/datagen/generator/UniformPostGenerator.java

@@ -40,7 +40,7 @@ protected PostInfo generatePostInfo( Random randomTag, Random randomDate, final
 		    }
 	    }
 	    postInfo.date = Dictionaries.dates.randomDate(randomDate,membership.creationDate()+DatagenParams.deltaTime);
-	    if( postInfo.date > Dictionaries.dates.getEndDateTime() ) return null;
+	    //if( postInfo.date > Dictionaries.dates.getEndDateTime() ) return null;
 	    return postInfo;
     }
 }