diff --git a/warehouse/ingest-scripts/src/main/resources/bin/ingest/bulk-ingest.sh b/warehouse/ingest-scripts/src/main/resources/bin/ingest/bulk-ingest.sh index fe30703f428..7936e750c9c 100755 --- a/warehouse/ingest-scripts/src/main/resources/bin/ingest/bulk-ingest.sh +++ b/warehouse/ingest-scripts/src/main/resources/bin/ingest/bulk-ingest.sh @@ -72,9 +72,9 @@ export HADOOP_OPTS="-Dfile.encoding=UTF8 -Duser.timezone=GMT $HADOOP_INGEST_OPTS export CHILD_MAP_OPTS="-Xmx${BULK_CHILD_MAP_MAX_MEMORY_MB}m -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF8 -Duser.timezone=GMT -XX:+UseNUMA $CHILD_INGEST_OPTS" export CHILD_REDUCE_OPTS="-Xmx${BULK_CHILD_REDUCE_MAX_MEMORY_MB}m -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF8 -Duser.timezone=GMT -XX:+UseNUMA $CHILD_INGEST_OPTS" -echo $INGEST_HADOOP_HOME/bin/hadoop jar ${DATAWAVE_INGEST_CORE_JAR} datawave.ingest.mapreduce.job.IngestJob -jt $INGEST_JOBTRACKER_NODE $INPUT_FILES ${INGEST_CONFIG[@]} -cacheBaseDir $JOB_CACHE_DIR -cacheJars $LIBJARS -user $USERNAME -pass $PASSWORD -instance $WAREHOUSE_INSTANCE_NAME -zookeepers $WAREHOUSE_ZOOKEEPERS -workDir $WORKDIR -flagFileDir ${FLAG_DIR} -flagFilePattern '.*_(bulk)_.*\.flag' -srcHdfs $INGEST_HDFS_NAME_NODE -destHdfs $WAREHOUSE_HDFS_NAME_NODE -distCpConfDir $WAREHOUSE_HADOOP_CONF -mapred.map.child.java.opts=\"$CHILD_MAP_OPTS\" -mapred.reduce.child.java.opts=\"$CHILD_REDUCE_OPTS\" "${BATCHWRITER_OPTS}" $MAPRED_OPTS $EXTRA_OPTS +echo $INGEST_HADOOP_HOME/bin/hadoop jar ${DATAWAVE_INGEST_CORE_JAR} datawave.ingest.mapreduce.job.IngestJob -jt $INGEST_JOBTRACKER_NODE $INPUT_FILES ${INGEST_CONFIG[@]} -cacheBaseDir $JOB_CACHE_DIR -cacheJars $LIBJARS -user $USERNAME -pass $PASSWORD -instance $WAREHOUSE_INSTANCE_NAME -zookeepers $WAREHOUSE_ZOOKEEPERS -workDir $WORKDIR -flagFileDir ${FLAG_DIR} -flagFilePattern '.*_(bulk)_.*\.flag' -srcHdfs $INGEST_HDFS_NAME_NODE -destHdfs $WAREHOUSE_HDFS_NAME_NODE -distCpConfDir $WAREHOUSE_HADOOP_CONF -mapreduce.map.java.opts=\"$CHILD_MAP_OPTS\" -mapreduce.reduce.java.opts=\"$CHILD_REDUCE_OPTS\" "${BATCHWRITER_OPTS}" $MAPRED_OPTS $EXTRA_OPTS -$INGEST_HADOOP_HOME/bin/hadoop jar ${DATAWAVE_INGEST_CORE_JAR} datawave.ingest.mapreduce.job.IngestJob -jt $INGEST_JOBTRACKER_NODE $INPUT_FILES ${INGEST_CONFIG[@]} -cacheBaseDir $JOB_CACHE_DIR -cacheJars $LIBJARS -user $USERNAME -pass $PASSWORD -instance $WAREHOUSE_INSTANCE_NAME -zookeepers $WAREHOUSE_ZOOKEEPERS -workDir $WORKDIR -flagFileDir ${FLAG_DIR} -flagFilePattern '.*_(bulk)_.*\.flag' -srcHdfs $INGEST_HDFS_NAME_NODE -destHdfs $WAREHOUSE_HDFS_NAME_NODE -distCpConfDir $WAREHOUSE_HADOOP_CONF -mapred.map.child.java.opts="$CHILD_MAP_OPTS" -mapred.reduce.child.java.opts="$CHILD_REDUCE_OPTS" "${BATCHWRITER_OPTS}" $MAPRED_OPTS $EXTRA_OPTS +$INGEST_HADOOP_HOME/bin/hadoop jar ${DATAWAVE_INGEST_CORE_JAR} datawave.ingest.mapreduce.job.IngestJob -jt $INGEST_JOBTRACKER_NODE $INPUT_FILES ${INGEST_CONFIG[@]} -cacheBaseDir $JOB_CACHE_DIR -cacheJars $LIBJARS -user $USERNAME -pass $PASSWORD -instance $WAREHOUSE_INSTANCE_NAME -zookeepers $WAREHOUSE_ZOOKEEPERS -workDir $WORKDIR -flagFileDir ${FLAG_DIR} -flagFilePattern '.*_(bulk)_.*\.flag' -srcHdfs $INGEST_HDFS_NAME_NODE -destHdfs $WAREHOUSE_HDFS_NAME_NODE -distCpConfDir $WAREHOUSE_HADOOP_CONF -mapreduce.map.java.opts="$CHILD_MAP_OPTS" -mapreduce.reduce.java.opts="$CHILD_REDUCE_OPTS" "${BATCHWRITER_OPTS}" $MAPRED_OPTS $EXTRA_OPTS RETURN_CODE=$? diff --git a/warehouse/ingest-scripts/src/main/resources/bin/ingest/live-ingest.sh b/warehouse/ingest-scripts/src/main/resources/bin/ingest/live-ingest.sh index cf7d49007cd..738b4a12ad2 100755 --- a/warehouse/ingest-scripts/src/main/resources/bin/ingest/live-ingest.sh +++ b/warehouse/ingest-scripts/src/main/resources/bin/ingest/live-ingest.sh @@ -71,10 +71,10 @@ export CHILD_MAP_OPTS="-Xmx${LIVE_CHILD_MAP_MAX_MEMORY_MB}m -XX:+UseConcMarkSwee export CHILD_REDUCE_OPTS="-Xmx${LIVE_CHILD_REDUCE_MAX_MEMORY_MB}m -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF8 -Duser.timezone=GMT -XX:+UseNUMA $CHILD_INGEST_OPTS" -echo $INGEST_HADOOP_HOME/bin/hadoop jar ${DATAWAVE_INGEST_CORE_JAR} datawave.ingest.mapreduce.job.IngestJob -jt $INGEST_JOBTRACKER_NODE $INPUT_FILES ${INGEST_CONFIG[@]} -cacheBaseDir $JOB_CACHE_DIR -cacheJars $LIBJARS -user $USERNAME -pass $PASSWORD -instance $WAREHOUSE_INSTANCE_NAME -zookeepers $WAREHOUSE_ZOOKEEPERS -workDir $WORKDIR -flagFileDir ${FLAG_DIR} -flagFilePattern '.*_live_.*\.flag' -mapred.map.child.java.opts=\"$CHILD_MAP_OPTS\" -mapred.reduce.child.java.opts=\"$CHILD_REDUCE_OPTS\" "${BATCHWRITER_OPTS}" $MAPRED_OPTS -outputMutations -mapOnly -srcHdfs $INGEST_HDFS_NAME_NODE -destHdfs $WAREHOUSE_HDFS_NAME_NODE $EXTRA_OPTS +echo $INGEST_HADOOP_HOME/bin/hadoop jar ${DATAWAVE_INGEST_CORE_JAR} datawave.ingest.mapreduce.job.IngestJob -jt $INGEST_JOBTRACKER_NODE $INPUT_FILES ${INGEST_CONFIG[@]} -cacheBaseDir $JOB_CACHE_DIR -cacheJars $LIBJARS -user $USERNAME -pass $PASSWORD -instance $WAREHOUSE_INSTANCE_NAME -zookeepers $WAREHOUSE_ZOOKEEPERS -workDir $WORKDIR -flagFileDir ${FLAG_DIR} -flagFilePattern '.*_live_.*\.flag' -mapreduce.map.java.opts=\"$CHILD_MAP_OPTS\" -mapreduce.reduce.java.opts=\"$CHILD_REDUCE_OPTS\" "${BATCHWRITER_OPTS}" $MAPRED_OPTS -outputMutations -mapOnly -srcHdfs $INGEST_HDFS_NAME_NODE -destHdfs $WAREHOUSE_HDFS_NAME_NODE $EXTRA_OPTS echo "For decreased latency, one can add the -mapOnly flag at the cost of possibly overcounting duplicate records" -$INGEST_HADOOP_HOME/bin/hadoop jar ${DATAWAVE_INGEST_CORE_JAR} datawave.ingest.mapreduce.job.IngestJob -jt $INGEST_JOBTRACKER_NODE $INPUT_FILES ${INGEST_CONFIG[@]} -cacheBaseDir $JOB_CACHE_DIR -cacheJars $LIBJARS -user $USERNAME -pass $PASSWORD -instance $WAREHOUSE_INSTANCE_NAME -zookeepers $WAREHOUSE_ZOOKEEPERS -workDir $WORKDIR -flagFileDir ${FLAG_DIR} -flagFilePattern '.*_live_.*\.flag' -mapred.map.child.java.opts="$CHILD_MAP_OPTS" -mapred.reduce.child.java.opts="$CHILD_REDUCE_OPTS" "${BATCHWRITER_OPTS}" $MAPRED_OPTS -outputMutations -mapOnly -srcHdfs $INGEST_HDFS_NAME_NODE -destHdfs $WAREHOUSE_HDFS_NAME_NODE $EXTRA_OPTS +$INGEST_HADOOP_HOME/bin/hadoop jar ${DATAWAVE_INGEST_CORE_JAR} datawave.ingest.mapreduce.job.IngestJob -jt $INGEST_JOBTRACKER_NODE $INPUT_FILES ${INGEST_CONFIG[@]} -cacheBaseDir $JOB_CACHE_DIR -cacheJars $LIBJARS -user $USERNAME -pass $PASSWORD -instance $WAREHOUSE_INSTANCE_NAME -zookeepers $WAREHOUSE_ZOOKEEPERS -workDir $WORKDIR -flagFileDir ${FLAG_DIR} -flagFilePattern '.*_live_.*\.flag' -mapreduce.map.java.opts="$CHILD_MAP_OPTS" -mapreduce.reduce.java.opts="$CHILD_REDUCE_OPTS" "${BATCHWRITER_OPTS}" $MAPRED_OPTS -outputMutations -mapOnly -srcHdfs $INGEST_HDFS_NAME_NODE -destHdfs $WAREHOUSE_HDFS_NAME_NODE $EXTRA_OPTS RETURN_CODE=$? diff --git a/web-services/map-reduce/src/test/resources/datawave/mapreduce/MapReduceJobs.xml b/web-services/map-reduce/src/test/resources/datawave/mapreduce/MapReduceJobs.xml index b8b4fc5d158..c5fa0ba6742 100644 --- a/web-services/map-reduce/src/test/resources/datawave/mapreduce/MapReduceJobs.xml +++ b/web-services/map-reduce/src/test/resources/datawave/mapreduce/MapReduceJobs.xml @@ -58,7 +58,7 @@ - +