Add a boolean type config item: upload_video_only

README.md

@@ -79,16 +79,17 @@ If you have a subscription of Microsoft OneDrive Stroage or Microsoft 365 Develo
 |    tenant_id | "" | for personal account, set it as "consumers"; for tenant account, set a specific tenant id
 |    scope | https://graph.microsoft.com/.default | not required
 |    video_root_folder | yihack_videos |  name string without white spaces
+|    upload_video_only | true | not required; set it false will upload *.jpg files in the record folders
 |    auto_clean_threshold | 100 |  value in range [50, 100) will enable this feature
 |    enable_idle_transfer | false |  setting to true has chances of files upload delayed
 
 
-3. upload code and dependent files to your camera sd card via `ssh` with `root` account or a FTP tool, the target path: `/tmp/sd/yi-hack`:
+1. upload code and dependent files to your camera sd card via `ssh` with `root` account or a FTP tool, the target path: `/tmp/sd/yi-hack`:
    - upload `curl` and `jq` binaries from local `bin` directory to `/tmp/sd/yi-hack/sbin`
    - upload your own `config.json`file, `init.sh`, `stop.sh` and `scripts` directory to `/tmp/sd/yi-hack/onedrive`
 
-4. sign in your [Microsoft Azure](https://login.microsoftonline.com/) account first
-5. run the entry Shell script `init.sh` to complete the application authorization grant flow
+2. sign in your [Microsoft Azure](https://login.microsoftonline.com/) account first
+3. run the entry Shell script `init.sh` to complete the application authorization grant flow
 
 ```bash
 cd /tmp/sd/yihack/onedrive/
@@ -110,14 +111,14 @@ cd /tmp/sd/yihack/onedrive/
 cd /tmp/sd/yi-hack/onedrive
 ./stop.sh
 ```
-- You might have messed up the configuration or you are not content the running situation, you want to start again from the beginning.
+- You might have messed up the configuration or you are not content with the running situation, you want to start again from the beginning.
   - first, stop the uploader first by following above
-  - delete all generated data (or keep file `data/last_upload.json` if you want the uploader to continue your uploading without any repetitive uploads).
+  - delete all generated data (or keep file `data/last_upload.json` if you want the uploader to resume your uploading without any repetitive uploads).
 ```bash
 rm -rf data
 rm -rm log
 ```
-  - If you want to setup with another OneDrive account( or Azure application data), you can delete `config.json` or replace it with a new one as well.
+  - If you want to setup with another OneDrive account (or Azure application data), you can delete or edit your current `config.json`, or replace it with a new one as well.
 
 
 

init.sh

@@ -10,6 +10,7 @@ init_globals() {
 	camera_idled=false 
 	default_video_root_folder="yihack_videos"
 	query=''; resp=''; error=''; video_root_folder=''
+	upload_video_only=true # upload mp4 files only
 	auto_clean_threshold=100  # disable the auto-clean feature
 	DRIVE_BASE_URI="https://graph.microsoft.com/v1.0/me/drive"
 	SD_RECORD_ROOT="/tmp/sd/record"
@@ -25,6 +26,11 @@ init_globals() {
 		color_print "BROWN" "A template config.json file is generated for you, please fill in it and try again."
 		exit 0
 	else
+		local upload_file_type=$(jq --raw-output '.upload_video_only' config.json)
+		if [ ! -z "${upload_file_type}" ] && [ "${upload_file_type}" != "null" ]; then
+			upload_video_only=${upload_file_type}
+		fi
+
 		local threshold=$(jq --raw-output '.auto_clean_threshold' config.json)
 		if [ ! -z "${threshold}" ] && [ "${threshold}" != "null" ]; then
 			auto_clean_threshold=${threshold}

scripts/api.sh

@@ -12,7 +12,7 @@ send_api_request() {
 		write_log "$@, Error: $api_error_message, Code: ${api_error_code}"
 		if [ "${api_error_code}" = "InvalidAuthenticationToken" ] && [ $1 != "--retry" ]; then 
 			write_log "Token invalid or expired, start to renew the tokens..."
-			refresh_oauth2_tokens
+			refresh_oauth2_tokens "--onetime"
 			send_api_request "--retry" $@
 		fi 
 	else

scripts/oauth2.sh

@@ -115,7 +115,7 @@ manage_oauth2_tokens() {
 			refresh_token=$(jq --raw-output '.refresh_token' ./data/token.json)
 			if [ ! -z $access_token ] && [ ! -z $refresh_token ]; then 
 				color_print "GREEN" "Found an existing refresh token, start to test its availability..."
-				refresh_oauth2_tokens "test"
+				refresh_oauth2_tokens "--test"
 			fi		
 			if [ -z $error ] || [ "$error" = "null" ]; then
 				need_re_assign=false

scripts/upload.sh

@@ -46,6 +46,7 @@ manage_drive_auto_clean() {
 	local used; local total; local used_ratio
 	local need_auto_clean; local auto_clean_done=false 
 
+	write_log "Start the auto-clean monitor..."
 	while [ 1 ]; do 
 		get_drive_status
 		used=$(echo ${resp} | jq '.quota.used')
@@ -63,7 +64,7 @@ manage_drive_auto_clean() {
 			break # do nothing (do not start or haven't started auto-clean)
 		fi		
 
-		sleep $((30*60)) # check auto-clean every 30 minutes
+		sleep $((10*60)) # check auto-clean every 10 minutes
 	done 
 }
 
@@ -147,28 +148,50 @@ get_one_file_to_upload() {
 # return: files list by modified time from past to current
 build_media_file_index() {	
 	if [ $# -eq 0 ]; then
-		echo "Build files uploading index..." >> log/logs
+		write_log "Build files uploading index..."
 		# files only sort by mtime in separate direcotries, cannot assure be sorted all by file mtime
 		# find /tmp/sd/record -maxdepth 2 -type f \( -iname \*.jpg -o -iname \*.mp4 \) | xargs ls -1rt > ./data/files.index 
 		# ls -1rtR /tmp/sd/record/*/ | awk '{ gsub("\:", ""); if ($1 ~ /sd/) { dir=$1 } else if(length($1) > 0) { printf "%s%s\n", dir, $1} }'
-		ls -1R /tmp/sd/record/*/ | awk '{ gsub("\:", ""); if ($1 ~ /sd/) { dir=$1 } else if(length($1) > 0) { printf "%s%s\n", dir, $1} }' > ./data/files.index 
+		if [ ${upload_video_only} != true ]; then  
+			ls -1R /tmp/sd/record/*/ | awk '{ gsub("\:", ""); if ($1 ~ /sd/) { dir=$1 } else if(length($1) > 0) { printf "%s%s\n", dir, $1} }' > ./data/files.index 
+		else 
+			ls -1R /tmp/sd/record/*/*.mp4 > ./data/files.index 
+		fi 
 	else
-		# local last_uploaded_file_ts=$(get_file_created_timestamp $1)
-		# local current_time_ts=$(date +%s)
-		# local eclipsed_mins=$(((${current_time_ts}-${last_uploaded_file_ts})/60))		
-		# local eclipsed_mins=$(get_elipsed_minutes ${last_uploaded_file_ts})
-
-		echo "Build a new uploading index for files created later than file ${1}..." >> log/logs
-		# Use -newer FILE is simple
-		# find /tmp/sd/record -maxdepth 2 -type f \( -iname \*.jpg -o -iname \*.mp4 \) \
-		# -mmin -${eclipsed_mins} | xargs ls -1rt > ./data/files.index  # not fully sorted by mtime (by directory first)
+		write_log "Build a new uploading index for files created later than file ${1}..."
+
+		local file_parent=$(parse_file_parent ${last_uploaded}) 
+		if [ ! -d ${file_parent} ]; then 
+			local last_uploaded_file_ts=$(get_file_created_timestamp $1)
+			local current_time_ts=$(date +%s)
+			local eclipsed_mins=$(((${current_time_ts}-${last_uploaded_file_ts})/60))		
+			local eclipsed_mins=$(get_elipsed_minutes ${last_uploaded_file_ts})
+
+			if [ ${upload_video_only} != true ]; then 
+				find /tmp/sd/record/ -mindepth 1 -type d -mmin -${eclipsed_mins} | xargs ls -1R | \
+				awk '{ gsub("\:", ""); if ($1 ~ /sd/) { dir=$1 } else if(length($1) > 0) { printf "%s%s\n", dir, $1} }' \
+				> ./data/files.index 
+			else 
+				find /tmp/sd/record/ -mindepth 1 -type d -mmin -${eclipsed_mins} | xargs ls -1R | \
+				awk '{ gsub("\:", ""); if ($1 ~ /sd/) { dir=$1 } else if($1 ~ /mp4/) { printf "%s%s\n", dir, $1} }' \
+				> ./data/files.index
+			fi 
+		else 
+			if [ ${upload_video_only} != true ]; then 
+				find /tmp/sd/record/ -mindepth 1 -type d -newer /tmp/sd/record/${file_parent} | xargs ls -1R | \
+				awk '{ gsub("\:", ""); if ($1 ~ /sd/) { dir=$1 } else if(length($1) > 0) { printf "%s%s\n", dir, $1} }' \
+				> ./data/files.index
+			else 
+				find /tmp/sd/record/ -mindepth 1 -type d -newer /tmp/sd/record/${file_parent} | xargs ls -1R | \
+				awk '{ gsub("\:", ""); if ($1 ~ /sd/) { dir=$1 } else if($1 ~ /mp4/) { printf "%s%s\n", dir, $1} }' \
+				> ./data/files.index
+			fi
+		fi
 
-		local file_parent=$(parse_file_parent ${last_uploaded})
-		find /tmp/sd/record/ -mindepth 1 -type d -newer /tmp/sd/record/${file_parent} | xargs ls -1R | \
-		awk '{ gsub("\:", ""); if ($1 ~ /sd/) { dir=$1 } else if(length($1) > 0) { printf "%s%s\n", dir, $1} }' \
-		> ./data/files.index
+		rm data/last_upload.json  # need to create new last_upload.json file
 	fi 
-	echo "Build the files uploading index successfully." >> ./log/logs
+
+	write_log "Build the files uploading index successfully."
 	using_fileindex=true
 }
 
@@ -244,17 +267,26 @@ get_next_file() {
 		# check if there is a newer file in the same folder (using `ls -1` can save awk '{print $9}')
 		# this will return a filename without the path included
 		echo "Search next file from the same directory: ${file_parent}" >> ./log/next_file
-		next_file=$(ls -l /tmp/sd/record/${file_parent} | grep -A1 ${file_name} | awk '{print $9}' | grep -v ${file_name})
+		if [ ${upload_video_only} != true ]; then 
+			next_file=$(ls -1 /tmp/sd/record/${file_parent} | grep -A1 ${file_name} | grep -v ${file_name}) # filename only
+			if [ ! -z "${next_file}" ]; then 
+				next_file="/tmp/sd/record/${file_parent}/${next_file}"
+			fi 
+		else
+			next_file=$(ls -1 /tmp/sd/record/${file_parent}/*.mp4 | grep -A1 ${file_name} | grep -v ${file_name}) # full path
+		fi
 
-		if [ -z "${next_file}" ]; then
-			# check newer file from another newer folder
+		# check newer file from another newer folder
+		if [ -z "${next_file}" ]; then			
 			local next_folder=$(get_next_folder ${file_parent})
 			echo "Can not find a file to upload, search another folder: ${next_folder}" >> ./log/next_file
 			if [ ! -z "${next_folder}" ]; then 
-				next_file=$(find ${next_folder} -type f | awk 'FNR <= 1')	# first file in the folder with full path
+				if [ ${upload_video_only} != true ]; then 
+					next_file=$(find ${next_folder} -type f | awk 'FNR <= 1')	# first file in the folder with full path
+				else 
+					next_file=$(find ${next_folder} -type f -iname \*.mp4| awk 'FNR <= 1')	# first file in the folder with full path
+				fi
 			fi		
-		else
-			next_file="/tmp/sd/record/${file_parent}/${next_file}"
 		fi 
 	fi