diff --git a/mock/Template_Copy-Only/publisher/README.md b/mock/Template_Copy-Only/publisher/README.md
deleted file mode 100755
index 789a300a..00000000
--- a/mock/Template_Copy-Only/publisher/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-This is used as mock data on multiple "drives".
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/2 Cool (2020)/2 Cool - pg - 2h - 2017 Moms.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/2 Cool (2020)/2 Cool - pg - 2h - 2017 Moms.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/2 Cool (2020)/2 Cool - pg - 2h - 2017 Moms.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/2 Cool (2020)/2 Cool - pg - 2h - 2017 Moms.srt b/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/2 Cool (2020)/2 Cool - pg - 2h - 2017 Moms.srt
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/2 Cool (2020)/2 Cool - pg - 2h - 2017 Moms.srt
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A2 Cool Movie (2017)/A2 Cool Movie - pg - 2h - 2017 Moms BluRay.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A2 Cool Movie (2017)/A2 Cool Movie - pg - 2h - 2017 Moms BluRay.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A2 Cool Movie (2017)/A2 Cool Movie - pg - 2h - 2017 Moms BluRay.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A2 Cool Movie (2017)/Thumbs.db b/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A2 Cool Movie (2017)/Thumbs.db
deleted file mode 100755
index 56f3b36e..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A2 Cool Movie (2017)/Thumbs.db
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A2 Cool Movie (2017)/a2-movie-poster.jpg b/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A2 Cool Movie (2017)/a2-movie-poster.jpg
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A2 Cool Movie (2017)/a2-movie-poster.jpg
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A3 Cool Movie (2017)/A3 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A3 Cool Movie (2017)/A3 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A3 Cool Movie (2017)/A3 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A4 Cool Movie (2017)/A4 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A4 Cool Movie (2017)/A4 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A4 Cool Movie (2017)/A4 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A4 Cool Movie (2017)/a4-movie-poster.jpg b/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A4 Cool Movie (2017)/a4-movie-poster.jpg
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/A4 Cool Movie (2017)/a4-movie-poster.jpg
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/This_is_in_root_of_'A'_Movies b/mock/Template_Copy-Only/publisher/media-a/media/MyMovies/This_is_in_root_of_'A'_Movies
deleted file mode 100755
index e69de29b..00000000
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show Moms DVD- s01e01.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show Moms DVD- s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show Moms DVD- s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show Moms DVD- s01e02.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show Moms DVD- s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/A1 TV Show Moms DVD- s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/File in A1 TV Show Season 01.txt b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/File in A1 TV Show Season 01.txt
deleted file mode 100755
index f88744d5..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 01/File in A1 TV Show Season 01.txt
+++ /dev/null
@@ -1 +0,0 @@
-123456789.123456789.123456789.123456789.12
\ No newline at end of file
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e01.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e02.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e03.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e04.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A1 TV Show (2017)/Season 02/A1 TV Show - s02e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A2 TV Show (2017)/Season 01/A2 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-a/media/MyTVShows/A3 TV Show (2017)/A2 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/A3 Cool Movie (2017)/A3 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/A3 Cool Movie (2017)/A3 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/A3 Cool Movie (2017)/A3 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/B1 Cool Movie (2017)/B1 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/B1 Cool Movie (2017)/B1 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/B1 Cool Movie (2017)/B1 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/B2 Cool Movie (2017)/B2 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/B2 Cool Movie (2017)/B2 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/B2 Cool Movie (2017)/B2 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/B2 Cool Movie (2017)/b2-movie-poster.jpg b/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/B2 Cool Movie (2017)/b2-movie-poster.jpg
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreMovies/B2 Cool Movie (2017)/b2-movie-poster.jpg
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 01/B1 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e01.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e02.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e03.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e04.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B1 TV Show (2017)/Season 02/B1 TV Show - s02e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B2 TV Show (2017)/Season 01/B2 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-b/media/MoreTVShows/B3 TV Show (2017)/B3 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/README.md b/mock/Template_Copy-Only/publisher/media-c/README.md
deleted file mode 100755
index eab223a3..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-This is used as a mock target drive.
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C1 Cool Movie (2017)/C1 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C1 Cool Movie (2017)/C1 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C1 Cool Movie (2017)/C1 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C1 Cool Movie (2017)/c1-movie-poster.jpg b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C1 Cool Movie (2017)/c1-movie-poster.jpg
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C1 Cool Movie (2017)/c1-movie-poster.jpg
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C2 Cool Movie (2017)/C2 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C2 Cool Movie (2017)/C2 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C2 Cool Movie (2017)/C2 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C2 Cool Movie (2017)/c2-movie-poster.jpg b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C2 Cool Movie (2017)/c2-movie-poster.jpg
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreMovies/C2 Cool Movie (2017)/c2-movie-poster.jpg
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C1 TV Show (2017)/Season 01/C1 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/publisher/media-c/media/YetMoreTVShows/C2 TV Show (2017)/Season 01/C2 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/publisher/publisher-libraries.json b/mock/Template_Copy-Only/publisher/publisher-libraries.json
deleted file mode 100755
index 2a956654..00000000
--- a/mock/Template_Copy-Only/publisher/publisher-libraries.json
+++ /dev/null
@@ -1,48 +0,0 @@
-{
- "libraries": {
- "description": "Test Publisher",
- "host": "localhost:30000",
- "flavor": "windows",
- "terminal_allowed": "true",
- "key": "025e2ddb-942a-4206-8458-902a87e42e62",
- "case_sensitive": false,
- "ignore_patterns": [
- "desktop.ini",
- "Thumbs.db"
- ],
- "renaming": [
- {
- "from": "(?i) Moms dvd",
- "to": " DVD"
- },
- {
- "from": "(?i) Moms Bluray",
- "to": ""
- }
- ],
- "locations": [
- {
- "location": "TestRun/publisher",
- "minimum": "30GB"
- }
- ],
- "bibliography": [
- {
- "name": "Movies",
- "sources": [
- "TestRun/publisher/media-a/media/MyMovies",
- "TestRun/publisher/media-b/media/MoreMovies",
- "TestRun/publisher/media-c/media/YetMoreMovies"
- ]
- },
- {
- "name": "TV Shows",
- "sources": [
- "TestRun/publisher/media-a/media/MyTVShows",
- "TestRun/publisher/media-b/media/MoreTVShows",
- "TestRun/publisher/media-c/media/YetMoreTVShows"
- ]
- }
- ]
- }
-}
diff --git a/mock/Template_Copy-Only/publisher/publisher-mock.json b/mock/Template_Copy-Only/publisher/publisher-mock.json
deleted file mode 100755
index 68a4c0f6..00000000
--- a/mock/Template_Copy-Only/publisher/publisher-mock.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- "libraries": {
- "description": "Test Publisher",
- "host": "clavius:30000",
- "flavor": "windows",
- "terminal_allowed": "true",
- "key": "025e2ddb-942a-4206-8458-902a87e42e62",
- "case_sensitive": false,
- "ignore_patterns": [
- "desktop.ini",
- "Thumbs.db"
- ],
- "renaming": [
- {
- "from": "(?i) Moms dvd",
- "to": " DVD"
- },
- {
- "from": "(?i) Moms Bluray",
- "to": ""
- }
- ],
- "bibliography": [
- {
- "name": "Movies",
- "sources": [
- "TestRun/publisher/media-a/media/MyMovies",
- "TestRun/publisher/media-b/media/MoreMovies",
- "TestRun/publisher/media-c/media/YetMoreMovies"
- ]
- },
- {
- "name": "TV Shows",
- "sources": [
- "TestRun/publisher/media-a/media/MyTVShows",
- "TestRun/publisher/media-b/media/MoreTVShows",
- "TestRun/publisher/media-c/media/YetMoreTVShows"
- ]
- }
- ]
- }
-}
diff --git a/mock/Template_Copy-Only/subscriber-1/README.md b/mock/Template_Copy-Only/subscriber-1/README.md
deleted file mode 100755
index 789a300a..00000000
--- a/mock/Template_Copy-Only/subscriber-1/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-This is used as mock data on multiple "drives".
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A1 Cool Movie (2017)/A1 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A1 Cool Movie (2017)/A1 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A1 Cool Movie (2017)/A1 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A2 Cool Movie (2017)/A2 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A2 Cool Movie (2017)/A2 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A2 Cool Movie (2017)/A2 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A2 Cool Movie (2017)/a2-movie-poster.jpg b/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A2 Cool Movie (2017)/a2-movie-poster.jpg
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A2 Cool Movie (2017)/a2-movie-poster.jpg
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A4 Cool Movie (2017)/A4 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A4 Cool Movie (2017)/A4 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A4 Cool Movie (2017)/A4 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A4 Cool Movie (2017)/a4-movie-poster.jpg b/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A4 Cool Movie (2017)/a4-movie-poster.jpg
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/MoviesA/A4 Cool Movie (2017)/a4-movie-poster.jpg
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A1 TV Show (2017)/Season 01/A1 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A2 TV Show (2017)/Season 01/A2 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A2 TV Show (2017)/Season 01/A2 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A2 TV Show (2017)/Season 01/A2 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A2 TV Show (2017)/Season 01/A2 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A2 TV Show (2017)/Season 01/A2 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A2 TV Show (2017)/Season 01/A2 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A3 TV Show (2017)/A2 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A3 TV Show (2017)/A2 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A3 TV Show (2017)/A2 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A3 TV Show (2017)/A2 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A3 TV Show (2017)/A2 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-1/vids/TVShowsA/A3 TV Show (2017)/A2 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-b/media/MoviesB/B2 Cool Movie (2017)/B2 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/subscriber-1/media-b/media/MoviesB/B2 Cool Movie (2017)/B2 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-b/media/MoviesB/B2 Cool Movie (2017)/B2 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e01.mp4 b/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e02.mp4 b/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e03.mp4 b/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e04.mp4 b/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B1 TV Show (2017)/Season 02/B1 TV Show - s02e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B2 TV Show (2017)/Season 01/B2 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B2 TV Show (2017)/Season 01/B2 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B2 TV Show (2017)/Season 01/B2 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B2 TV Show (2017)/Season 01/B2 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B2 TV Show (2017)/Season 01/B2 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-b/media/TVShowsB/B2 TV Show (2017)/Season 01/B2 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/README.md b/mock/Template_Copy-Only/subscriber-1/media-c/README.md
deleted file mode 100755
index eab223a3..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-This is used as a mock target drive.
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/A4 Cool Movie (2017)/A4 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/A4 Cool Movie (2017)/A4 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/A4 Cool Movie (2017)/A4 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/A4 Cool Movie (2017)/a4-movie-poster.jpg b/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/A4 Cool Movie (2017)/a4-movie-poster.jpg
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/A4 Cool Movie (2017)/a4-movie-poster.jpg
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/C2 Cool Movie (2017)/C2 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/C2 Cool Movie (2017)/C2 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/C2 Cool Movie (2017)/C2 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/C3 Cool Movie (2017)/C3 Cool Movie - pg - 2h - 2017.mp4 b/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/C3 Cool Movie (2017)/C3 Cool Movie - pg - 2h - 2017.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/C3 Cool Movie (2017)/C3 Cool Movie - pg - 2h - 2017.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/C3 Cool Movie (2017)/c3-movie-poster.jpg b/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/C3 Cool Movie (2017)/c3-movie-poster.jpg
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/MoviesC/C3 Cool Movie (2017)/c3-movie-poster.jpg
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C1 TV Show (2017)/Season 01/C1 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C1 TV Show (2017)/Season 01/C1 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C1 TV Show (2017)/Season 01/C1 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C1 TV Show (2017)/Season 01/C1 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C1 TV Show (2017)/Season 01/C1 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C1 TV Show (2017)/Season 01/C1 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e01.mp4 b/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e01.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e01.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e02.mp4 b/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e02.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e02.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e03.mp4 b/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e03.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e03.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e04.mp4 b/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e04.mp4
deleted file mode 100755
index 8d1c8b69..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-c/media/TVShowsC/C3 TV Show (2017)/Season 01/C3 TV Show - s01e04.mp4
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-d/README.md b/mock/Template_Copy-Only/subscriber-1/media-d/README.md
deleted file mode 100755
index eab223a3..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-d/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-This is used as a mock target drive.
-
diff --git a/mock/Template_Copy-Only/subscriber-1/media-d/media/NewMovies/README.md b/mock/Template_Copy-Only/subscriber-1/media-d/media/NewMovies/README.md
deleted file mode 100755
index 75cf9c78..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-d/media/NewMovies/README.md
+++ /dev/null
@@ -1 +0,0 @@
- Placeholder for a target directory
diff --git a/mock/Template_Copy-Only/subscriber-1/media-d/media/NewTVShows/README.md b/mock/Template_Copy-Only/subscriber-1/media-d/media/NewTVShows/README.md
deleted file mode 100755
index 75cf9c78..00000000
--- a/mock/Template_Copy-Only/subscriber-1/media-d/media/NewTVShows/README.md
+++ /dev/null
@@ -1 +0,0 @@
- Placeholder for a target directory
diff --git a/mock/Template_Copy-Only/subscriber-1/subscriber-1-libraries.json b/mock/Template_Copy-Only/subscriber-1/subscriber-1-libraries.json
deleted file mode 100755
index 4ad27bdc..00000000
--- a/mock/Template_Copy-Only/subscriber-1/subscriber-1-libraries.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- "libraries": {
- "description": "Test Subscriber-1",
- "host": "localhost:30000",
- "flavor": "windows",
- "terminal_allowed": "true",
- "key": "94eb0321-8a3d-4c10-81e9-1d89733f6d64",
- "case_sensitive": false,
- "ignore_patterns": [
- "desktop.ini",
- "Thumbs.db"
- ],
- "locations": [
- {
- "location": "TestRun/subscriber-1",
- "minimum": "30GB"
- }
- ],
- "bibliography": [
- {
- "name": "Movies",
- "sources": [
- "TestRun/subscriber-1/media-1/vids/MoviesA",
- "TestRun/subscriber-1/media-b/media/MoviesB",
- "TestRun/subscriber-1/media-c/media/MoviesC",
- "TestRun/subscriber-1/media-d/media/NewMovies"
- ]
- },
- {
- "name": "TV Shows",
- "sources": [
- "TestRun/subscriber-1/media-1/vids/TVShowsA",
- "TestRun/subscriber-1/media-b/media/TVShowsB",
- "TestRun/subscriber-1/media-c/media/TVShowsC",
- "TestRun/subscriber-1/media-d/media/NewTVShows"
- ]
- }
- ]
- }
-}
diff --git a/mock/Template_Copy-Only/subscriber-mock.json b/mock/Template_Copy-Only/subscriber-mock.json
deleted file mode 100755
index fa87200c..00000000
--- a/mock/Template_Copy-Only/subscriber-mock.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "libraries": {
- "description": "Mock Subscriber",
- "host": "rockplex:30000",
- "flavor": "linux",
- "terminal_allowed": "true",
- "key": "94eb0321-8a3d-4c10-81e9-1d89733f6d64",
- "case_sensitive": false,
- "ignore_patterns": [
- "desktop.ini",
- "Thumbs.db"
- ],
- "bibliography": [
- {
- "name": "Movies",
- "sources": [
- "subscriber/media-1/vids/MoviesA",
- "subscriber/media-b/media/MoviesB",
- "subscriber/media-c/media/MoviesC",
- "subscriber/media-d/media/NewMovies"
- ]
- },
- {
- "name": "TV Shows",
- "sources": [
- "subscriber/media-1/vids/TVShowsA",
- "subscriber/media-b/media/TVShowsB",
- "subscriber/media-c/media/TVShowsC",
- "subscriber/media-d/media/NewTVShows"
- ]
- }
- ]
- }
-}
diff --git a/mock/Template_Copy-Only/targets-1.json b/mock/Template_Copy-Only/targets-1.json
deleted file mode 100755
index 2b6b27ba..00000000
--- a/mock/Template_Copy-Only/targets-1.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "targets": {
- "description": "Subscriber Targets",
- "storage": [
- {
- "name": "Movies",
- "minimum": "50kb",
- "locations": [
- "TestRun/subscriber-1/media-d/media/NewMovies"
- ]
- },
- {
- "name": "TV Shows",
- "minimum": "1GB",
- "locations": [
- "TestRun/subscriber-1/media-d/media/NewTVShows"
- ]
- }
- ]
- }
-}
diff --git a/mock/Template_Copy-Only/targets-mock.json b/mock/Template_Copy-Only/targets-mock.json
deleted file mode 100755
index 8ac2571d..00000000
--- a/mock/Template_Copy-Only/targets-mock.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "targets": {
- "description": "Targets",
- "storage": [
- {
- "name": "Movies",
- "minimum": "50kb",
- "locations": [
- "subscriber/media-d/media/NewMovies"
- ]
- },
- {
- "name": "TV Shows",
- "minimum": "1GB",
- "locations": [
- "subscriber/media-d/media/NewTVShows"
- ]
- }
- ]
- }
-}
diff --git a/mock/compare-clean.bat b/mock/compare-clean.bat
deleted file mode 100755
index cbff1f00..00000000
--- a/mock/compare-clean.bat
+++ /dev/null
@@ -1,16 +0,0 @@
-@echo off
-REM compare-clear
-REM Useful for doing TestRun directory compares
-
-set base=%~dp0
-cd /d %base%
-
-if not exist .\TestRun goto NoDir
-
-del /s TestRun\*export.json
-del /s TestRun\*received*
-del /s TestRun\*generated*
-del /s TestRun\*.log
-del /s TestRun\*.txt
-
-:NoDir
diff --git a/mock/compare-copy.bat b/mock/compare-copy.bat
deleted file mode 100755
index 0178286f..00000000
--- a/mock/compare-copy.bat
+++ /dev/null
@@ -1,16 +0,0 @@
-@echo off
-REM compare-copy
-
-set base=%~dp0
-cd /d %base%
-
-if not exist .\TestRun goto NoDir
-
-if exist .\TestRun-compare rmdir /q /s .\TestRun-compare
-
-move TestRun TestRun-compare
-
-echo/
-echo Run reset.bat to setup a new TestRun directory
-
-:NoDir
diff --git a/mock/media-base_copy-only/hints/hint-server.json b/mock/media-base_copy-only/hints/hint-server.json
new file mode 100644
index 00000000..edeab8ed
--- /dev/null
+++ b/mock/media-base_copy-only/hints/hint-server.json
@@ -0,0 +1,26 @@
+{
+ "libraries": {
+ "description": "Hint Server",
+ "host": "localhost:50971",
+ "listen": "localhost:50971",
+ "flavor": "linux",
+ "terminal_allowed": "false",
+ "key": "dd154b96-9dbe-43ad-be0f-ceaa00d4a64e",
+ "case_sensitive": "false",
+ "locations": [
+ {
+ "location": "test/hints/datastore",
+ "minimum": "42GB"
+ }
+ ],
+ "bibliography": [
+ {
+ "name": "ELS Hint Status Datastore",
+ "sources": [
+ "test/hints/datastore"
+ ]
+ }
+ ]
+ }
+}
+
diff --git a/mock/media-base_copy-only/publisher/media/Movies/A Bugs Show (2021)/Season 2/Bugs What An Interview - S02E01.mp4 b/mock/media-base_copy-only/publisher/media/Movies/A Bugs Show (2021)/Season 2/Bugs What An Interview - S02E01.mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/Movies/A Bugs Show (2021)/Season 2/Bugs What An Interview - S02E01.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/Movies/Hubble (2021)/Hubble.mp4 b/mock/media-base_copy-only/publisher/media/Movies/Hubble (2021)/Hubble.mp4
new file mode 100644
index 00000000..1e1702e6
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/Movies/Hubble (2021)/Hubble.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/Movies/Hubble (2021)/cover.png b/mock/media-base_copy-only/publisher/media/Movies/Hubble (2021)/cover.png
new file mode 100644
index 00000000..1b1d5918
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/Movies/Hubble (2021)/cover.png differ
diff --git a/mock/media-base_copy-only/publisher/media/Movies/Hubble (2021)/delete-inline.els b/mock/media-base_copy-only/publisher/media/Movies/Hubble (2021)/delete-inline.els
new file mode 100644
index 00000000..7ddfcdbb
--- /dev/null
+++ b/mock/media-base_copy-only/publisher/media/Movies/Hubble (2021)/delete-inline.els
@@ -0,0 +1,7 @@
+# Delete inline file test
+For MediaServer
+For BackupOne
+For BackupTwo
+
+rm "cover.png"
+
diff --git a/mock/media-base_copy-only/publisher/media/Movies/Launch to Mars (2021)/Lander to Mars.mp4 b/mock/media-base_copy-only/publisher/media/Movies/Launch to Mars (2021)/Lander to Mars.mp4
new file mode 100644
index 00000000..e7a19c3f
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/Movies/Launch to Mars (2021)/Lander to Mars.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/Movies/Launch to Mars (2021)/rename-inline.els b/mock/media-base_copy-only/publisher/media/Movies/Launch to Mars (2021)/rename-inline.els
new file mode 100644
index 00000000..bd53a6ad
--- /dev/null
+++ b/mock/media-base_copy-only/publisher/media/Movies/Launch to Mars (2021)/rename-inline.els
@@ -0,0 +1,7 @@
+# Rename inline file test
+For MediaServer
+For BackupOne
+For BackupTwo
+
+mv "Lander to Mars.mp4" "Launch to Mars.mp4"
+
diff --git a/mock/media-base_copy-only/publisher/media/Movies/Of Outer Space (2021)/Outer Space.mp4 b/mock/media-base_copy-only/publisher/media/Movies/Of Outer Space (2021)/Outer Space.mp4
new file mode 100644
index 00000000..4efb70d5
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/Movies/Of Outer Space (2021)/Outer Space.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/Movies/Satellite Repair (2021)/Satellite Repair.mp4 b/mock/media-base_copy-only/publisher/media/Movies/Satellite Repair (2021)/Satellite Repair.mp4
new file mode 100644
index 00000000..06d4f7c7
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/Movies/Satellite Repair (2021)/Satellite Repair.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/Movies/delete-directory.els b/mock/media-base_copy-only/publisher/media/Movies/delete-directory.els
new file mode 100644
index 00000000..09b2ac1e
--- /dev/null
+++ b/mock/media-base_copy-only/publisher/media/Movies/delete-directory.els
@@ -0,0 +1,7 @@
+# Delete directory with subdirectories test
+For MediaServer
+For BackupOne
+For BackupTwo
+
+rm "A Bugs Show (2021)"
+
diff --git a/mock/media-base_copy-only/publisher/media/Movies/rename-directory.els b/mock/media-base_copy-only/publisher/media/Movies/rename-directory.els
new file mode 100644
index 00000000..7f0e4f84
--- /dev/null
+++ b/mock/media-base_copy-only/publisher/media/Movies/rename-directory.els
@@ -0,0 +1,7 @@
+# Rename directory test
+For MediaServer
+For BackupOne
+For BackupTwo
+
+mv "Of Outer Space (2021)" "Outer Space (2021)"
+
diff --git a/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs An Interview - S01E01.mp4 b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs An Interview - S01E01.mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs An Interview - S01E01.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs Other Interview eps 2.mp4 b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs Other Interview eps 2.mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs Other Interview eps 2.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 1/rename-inline-in-subdir.els b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 1/rename-inline-in-subdir.els
new file mode 100644
index 00000000..ee72df51
--- /dev/null
+++ b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 1/rename-inline-in-subdir.els
@@ -0,0 +1,7 @@
+# Rename inline in subdir file test
+For MediaServer
+For BackupOne
+For BackupTwo
+
+mv "Bugs Other Interview eps 2.mp4" "Bugs Another Interview - S01E02.mp4
+
diff --git a/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 2/Bugs What A Repeated Interview - S02E01 .mp4 b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 2/Bugs What A Repeated Interview - S02E01 .mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/Season 2/Bugs What A Repeated Interview - S02E01 .mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/What Another Bugs Interview.mp4 b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/What Another Bugs Interview.mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/What Another Bugs Interview.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/move-inline-file.els b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/move-inline-file.els
new file mode 100644
index 00000000..39c18c9d
--- /dev/null
+++ b/mock/media-base_copy-only/publisher/media/TV Shows/A Bugs Show (2021)/move-inline-file.els
@@ -0,0 +1,7 @@
+# Move inline file test
+For MediaServer
+For BackupOne
+For BackupTwo
+
+mv "What Another Bugs Interview.mp4" "Season 1/Bugs What Another Interview - S01E03.mp4"
+
diff --git a/mock/media-base_copy-only/publisher/media/TV Shows/Of Outer Space (2021)/Outer Space.mp4 b/mock/media-base_copy-only/publisher/media/TV Shows/Of Outer Space (2021)/Outer Space.mp4
new file mode 100644
index 00000000..4efb70d5
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/TV Shows/Of Outer Space (2021)/Outer Space.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/TV Shows/Virus (2021)/Virus.mp4 b/mock/media-base_copy-only/publisher/media/TV Shows/Virus (2021)/Virus.mp4
new file mode 100644
index 00000000..d336d7e1
Binary files /dev/null and b/mock/media-base_copy-only/publisher/media/TV Shows/Virus (2021)/Virus.mp4 differ
diff --git a/mock/media-base_copy-only/publisher/media/TV Shows/move-directory-cross-library.els b/mock/media-base_copy-only/publisher/media/TV Shows/move-directory-cross-library.els
new file mode 100644
index 00000000..7536db21
--- /dev/null
+++ b/mock/media-base_copy-only/publisher/media/TV Shows/move-directory-cross-library.els
@@ -0,0 +1,7 @@
+# Move directory cross library test
+For MediaServer
+For BackupOne
+For BackupTwo
+
+mv "Virus (2021)" "Movies | Virus (2021)"
+
diff --git a/mock/media-base_copy-only/publisher/publisher.json b/mock/media-base_copy-only/publisher/publisher.json
new file mode 100644
index 00000000..86a978e8
--- /dev/null
+++ b/mock/media-base_copy-only/publisher/publisher.json
@@ -0,0 +1,38 @@
+{
+ "libraries": {
+ "description": "Media Publisher",
+ "host": "localhost:50271",
+ "listen": "localhost:50271",
+ "flavor": "linux",
+ "terminal_allowed": "true",
+ "key": "aa1673d4-5284-4a7d-86c9-3df20adace46",
+ "case_sensitive": "false",
+ "ignore_patterns": [
+ "(?i)desktop\\.ini",
+ ".*\\.fuse.*",
+ ".*\\.srt",
+ "Thumbs\\.db"
+ ],
+ "locations": [
+ {
+ "location": "test/publisher/media",
+ "minimum": "42GB"
+ }
+ ],
+ "bibliography": [
+ {
+ "name": "Movies",
+ "sources": [
+ "test/publisher/media/Movies"
+ ]
+ },
+ {
+ "name": "TV Shows",
+ "sources": [
+ "test/publisher/media/TV Shows"
+ ]
+ }
+ ]
+ }
+}
+
diff --git a/mock/media-base_copy-only/publisher/targets.json b/mock/media-base_copy-only/publisher/targets.json
new file mode 100644
index 00000000..265a7def
--- /dev/null
+++ b/mock/media-base_copy-only/publisher/targets.json
@@ -0,0 +1,22 @@
+{
+ "targets": {
+ "description": "Targets on Media Publisher",
+ "storage": [
+ {
+ "name": "Movies",
+ "minimum": "30GB",
+ "locations": [
+ "test/publisher/media/Movies"
+ ]
+ },
+ {
+ "name": "TV Shows",
+ "minimum": "30GB",
+ "locations": [
+ "test/publisher/media/TV Shows"
+ ]
+ }
+ ]
+ }
+}
+
diff --git a/mock/media-base_copy-only/subscriber-one/media/Movies/A Bugs Show (2021)/Season 2/Bugs What An Interview - S02E01.mp4 b/mock/media-base_copy-only/subscriber-one/media/Movies/A Bugs Show (2021)/Season 2/Bugs What An Interview - S02E01.mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/Movies/A Bugs Show (2021)/Season 2/Bugs What An Interview - S02E01.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-one/media/Movies/Hubble (2021)/Hubble.mp4 b/mock/media-base_copy-only/subscriber-one/media/Movies/Hubble (2021)/Hubble.mp4
new file mode 100644
index 00000000..1e1702e6
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/Movies/Hubble (2021)/Hubble.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-one/media/Movies/Hubble (2021)/cover.png b/mock/media-base_copy-only/subscriber-one/media/Movies/Hubble (2021)/cover.png
new file mode 100644
index 00000000..1b1d5918
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/Movies/Hubble (2021)/cover.png differ
diff --git a/mock/media-base_copy-only/subscriber-one/media/Movies/Launch to Mars (2021)/Lander to Mars.mp4 b/mock/media-base_copy-only/subscriber-one/media/Movies/Launch to Mars (2021)/Lander to Mars.mp4
new file mode 100644
index 00000000..e7a19c3f
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/Movies/Launch to Mars (2021)/Lander to Mars.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-one/media/Movies/Of Outer Space (2021)/Outer Space.mp4 b/mock/media-base_copy-only/subscriber-one/media/Movies/Of Outer Space (2021)/Outer Space.mp4
new file mode 100644
index 00000000..4efb70d5
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/Movies/Of Outer Space (2021)/Outer Space.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-one/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs An Interview - S01E01.mp4 b/mock/media-base_copy-only/subscriber-one/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs An Interview - S01E01.mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs An Interview - S01E01.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-one/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs Other Interview eps 2.mp4 b/mock/media-base_copy-only/subscriber-one/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs Other Interview eps 2.mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs Other Interview eps 2.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-one/media/TV Shows/A Bugs Show (2021)/What Another Bugs Interview.mp4 b/mock/media-base_copy-only/subscriber-one/media/TV Shows/A Bugs Show (2021)/What Another Bugs Interview.mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/TV Shows/A Bugs Show (2021)/What Another Bugs Interview.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-one/media/TV Shows/Of Outer Space (2021)/Outer Space.mp4 b/mock/media-base_copy-only/subscriber-one/media/TV Shows/Of Outer Space (2021)/Outer Space.mp4
new file mode 100644
index 00000000..4efb70d5
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/TV Shows/Of Outer Space (2021)/Outer Space.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-one/media/TV Shows/Virus (2021)/Virus.mp4 b/mock/media-base_copy-only/subscriber-one/media/TV Shows/Virus (2021)/Virus.mp4
new file mode 100644
index 00000000..d336d7e1
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-one/media/TV Shows/Virus (2021)/Virus.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-one/subscriber-one.json b/mock/media-base_copy-only/subscriber-one/subscriber-one.json
new file mode 100644
index 00000000..7c48cd46
--- /dev/null
+++ b/mock/media-base_copy-only/subscriber-one/subscriber-one.json
@@ -0,0 +1,38 @@
+{
+ "libraries": {
+ "description": "Subscriber One",
+ "host": "localhost:50371",
+ "listen": "localhost:50371",
+ "flavor": "linux",
+ "terminal_allowed": "true",
+ "key": "bb12a499-97a6-44e1-8a91-4cc898e45849",
+ "case_sensitive": "false",
+ "ignore_patterns": [
+ "(?i)desktop\\.ini",
+ ".*\\.fuse.*",
+ ".*\\.srt",
+ "Thumbs\\.db"
+ ],
+ "locations": [
+ {
+ "location": "test/subscriber-one/media",
+ "minimum": "42GB"
+ }
+ ],
+ "bibliography": [
+ {
+ "name": "Movies",
+ "sources": [
+ "test/subscriber-one/media/Movies"
+ ]
+ },
+ {
+ "name": "TV Shows",
+ "sources": [
+ "test/subscriber-one/media/TV Shows"
+ ]
+ }
+ ]
+ }
+}
+
diff --git a/mock/media-base_copy-only/subscriber-one/targets.json b/mock/media-base_copy-only/subscriber-one/targets.json
new file mode 100644
index 00000000..b8933730
--- /dev/null
+++ b/mock/media-base_copy-only/subscriber-one/targets.json
@@ -0,0 +1,22 @@
+{
+ "targets": {
+ "description": "Targets on Subscriber One",
+ "storage": [
+ {
+ "name": "Movies",
+ "minimum": "30GB",
+ "locations": [
+ "test/subscriber-one/media/Movies"
+ ]
+ },
+ {
+ "name": "TV Shows",
+ "minimum": "30GB",
+ "locations": [
+ "test/subscriber-one/media/TV Shows"
+ ]
+ }
+ ]
+ }
+}
+
diff --git a/mock/media-base_copy-only/subscriber-two/media/Movies/Hubble (2021)/Hubble.mp4 b/mock/media-base_copy-only/subscriber-two/media/Movies/Hubble (2021)/Hubble.mp4
new file mode 100644
index 00000000..1e1702e6
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-two/media/Movies/Hubble (2021)/Hubble.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-two/media/Movies/Of Outer Space (2021)/Outer Space.mp4 b/mock/media-base_copy-only/subscriber-two/media/Movies/Of Outer Space (2021)/Outer Space.mp4
new file mode 100644
index 00000000..4efb70d5
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-two/media/Movies/Of Outer Space (2021)/Outer Space.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-two/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs An Interview - S01E01.mp4 b/mock/media-base_copy-only/subscriber-two/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs An Interview - S01E01.mp4
new file mode 100644
index 00000000..eb9bccb4
Binary files /dev/null and b/mock/media-base_copy-only/subscriber-two/media/TV Shows/A Bugs Show (2021)/Season 1/Bugs An Interview - S01E01.mp4 differ
diff --git a/mock/media-base_copy-only/subscriber-two/subscriber-two.json b/mock/media-base_copy-only/subscriber-two/subscriber-two.json
new file mode 100644
index 00000000..c3a05c4d
--- /dev/null
+++ b/mock/media-base_copy-only/subscriber-two/subscriber-two.json
@@ -0,0 +1,38 @@
+{
+ "libraries": {
+ "description": "Subscriber Two",
+ "host": "localhost:50471",
+ "listen": "localhost:50471",
+ "flavor": "linux",
+ "terminal_allowed": "true",
+ "key": "cc155c0d-32d9-4640-8c20-b5fe6073e41d",
+ "case_sensitive": "false",
+ "ignore_patterns": [
+ "(?i)desktop\\.ini",
+ ".*\\.fuse.*",
+ ".*\\.srt",
+ "Thumbs\\.db"
+ ],
+ "locations": [
+ {
+ "location": "test/subscriber-two/media",
+ "minimum": "42GB"
+ }
+ ],
+ "bibliography": [
+ {
+ "name": "Movies",
+ "sources": [
+ "test/subscriber-two/media/Movies"
+ ]
+ },
+ {
+ "name": "TV Shows",
+ "sources": [
+ "test/subscriber-two/media/TV Shows"
+ ]
+ }
+ ]
+ }
+}
+
diff --git a/mock/media-base_copy-only/subscriber-two/targets.json b/mock/media-base_copy-only/subscriber-two/targets.json
new file mode 100644
index 00000000..9bde470c
--- /dev/null
+++ b/mock/media-base_copy-only/subscriber-two/targets.json
@@ -0,0 +1,22 @@
+{
+ "targets": {
+ "description": "Targets on Subscriber One",
+ "storage": [
+ {
+ "name": "Movies",
+ "minimum": "30GB",
+ "locations": [
+ "test/subscriber-two/media/Movies"
+ ]
+ },
+ {
+ "name": "TV Shows",
+ "minimum": "30GB",
+ "locations": [
+ "test/subscriber-two/media/TV Shows"
+ ]
+ }
+ ]
+ }
+}
+
diff --git a/mock/media-base_copy-only/test-hints.keys b/mock/media-base_copy-only/test-hints.keys
new file mode 100644
index 00000000..bd2d1f24
--- /dev/null
+++ b/mock/media-base_copy-only/test-hints.keys
@@ -0,0 +1,14 @@
+# ELS Hints keys
+#
+# Format: name uuid
+#
+# Name is any shortname, no spaces.
+# UUID is the key from the publisher.json file.
+#
+# Use space separators, no quotes.
+#
+
+MediaServer aa1673d4-5284-4a7d-86c9-3df20adace46
+BackupOne bb12a499-97a6-44e1-8a91-4cc898e45849
+BackupTwo cc155c0d-32d9-4640-8c20-b5fe6073e41d
+
diff --git a/mock/reset.bat b/mock/reset.bat
deleted file mode 100755
index 993c1e16..00000000
--- a/mock/reset.bat
+++ /dev/null
@@ -1,30 +0,0 @@
-@echo off
-REM reset [-f]
-
-set base=%~dp0
-cd /d %base%
-
-if not exist .\TestRun goto NoDir
-if "z%1" == "z-f" goto Execute
-echo/
-echo Reset TestRun Directory
-set r=
-set /P R=Confirm: DESTROY TestRun directory and recreate from templates (y/N)?
-if "z%R%" == "zy" goto Execute
-if "z%R%" == "zY" goto Execute
-goto Cancel
-
-:Execute
-rmdir /s /q .\TestRun
-if exist .\els.log del /q .\els.log
-
-:NoDir
-xcopy /I /E .\Template_Copy-Only .\TestRun
-echo Done
-goto JXT
-
-:Cancel
-echo Cancelled
-
-:JXT
-echo/
diff --git a/mock/run.bat b/mock/run.bat
deleted file mode 100755
index 6bc0624c..00000000
--- a/mock/run.bat
+++ /dev/null
@@ -1,6 +0,0 @@
-@echo off
-
-set base=%~dp0
-cd /d %base%
-
-java -cp "..\out\production\ELS\;..\lib;..\lib\*" com.groksoft.els.Main %*
diff --git a/mock/run.sh b/mock/run.sh
deleted file mode 100755
index 73f1c47e..00000000
--- a/mock/run.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-base=`dirname $0`
-cd ${base}
-
-java -cp "../out/production/Main/:../lib:../lib/*" com.groksoft.els.Main $*
diff --git a/mock/scripts/README.md b/mock/scripts/README.md
new file mode 100644
index 00000000..c6a88c59
--- /dev/null
+++ b/mock/scripts/README.md
@@ -0,0 +1,210 @@
+# Test Scripts
+
+Command-line scripts to test specific ELS application-level functionalities.
+
+*Note:* These are also an excellent example of the various ways ELS can be executed.
+
+ELS is composed of several different capabilties:
+
+ * Stand-alone back-up tool - the original
+ * Networked back-up tool
+ * Built-in SFTP
+ * Built-in STTY
+ * Built-in interactive terminals
+ * Local hint processing
+ * Networked hint processing
+ * Local Hint Tracker
+ * Networked Hint Status Server (Hint Status Server/HSS)
+
+
+## Test Organization
+
+Tests are *generally* organized in increasing options and functionality.
+
+ 00- Basic Functionality
+
+ 10- Local Backup
+
+ 20- Remote Backup
+
+ 30- Interactive terminals
+
+ 40- Local Hints
+
+ 50- Remote Hints
+
+ 60- Local Hint Tracker
+
+ 70- Remote Hint Server
+
+
+## Test Utility Scripts
+
+``clear-output.sh`` : Removes all files from the mock/output/ directory
+
+``reset.sh`` : Resets the mock/test/ directory by deleting it and copying the
+mock/media-base_copy-only directory to a new mock/test/ directory
+
+
+## Tests & Sequence
+
+Some of the following scripts are run once. Others are run multiple times to complete a particular test
+sequence.
+
+Examine the screen and/or log output for warnings, errors and exceptions. Examine the test/ directory
+for the appropriate changes at each step of the test sequences.
+
+Automation and result-checking have not been, and might not be, implemented. It's a lot of work.
+At this point it's a manual and visual process.
+
+### 00-00 Basic Functionality
+
+* ``00-01_Version.sh`` : Run once, does not create a log file
+
+* ``00-02_Bad-arguments.sh`` : Run once, should see an exception
+
+* ``00-03_Validate.sh`` : Run once
+
+* ``00-04_Export.sh`` : Run once
+
+* ``00-05_Duplicates.sh`` : No dupes
+
+* ``00-06_Duplicates-crosscheck.sh`` : Duplicate shown due to cross-check
+
+### 10-00 Local Backup
+
+* ``reset.sh`` : Reset the test/ directory
+
+* ``10-22_Backup-dryrun.sh`` : Run once
+
+* ``10-23_Backup.sh`` : Run once
+
+* ``reset.sh`` : Reset the test/ directory
+
+* ``10-24_Backup-exclude-lib.sh`` : Run once
+
+* ``reset.sh`` : Reset the test/ directory
+
+* ``10-25_Backup-include-lib.sh`` : Run once
+
+### 20-00 Remote Backup
+
+* ``reset.sh`` : Reset the test/ directory
+
+* ``20-21_Subscriber-listener.sh`` ; Separate terminal 1
+
+* ``20-22_Publisher-dryrun.sh`` : Separate terminal 2
+
+* ``20-21_Subscriber-listener.sh`` ; Separate terminal 1
+
+* ``20-23_Publisher-backup.sh`` : Separate terminal 2
+
+### 30-00 Interactive Terminals
+
+* ``reset.sh`` : Reset the test/ directory
+
+* ``30-21_Subscriber-listener.sh`` : Separate terminal 1, AbstractPollingIoProcessor$Processor warnings on quit
+
+* ``30-29_Publisher-manual.sh`` : Separate terminal 2
+
+* ``30-31_Publisher-listener.sh`` : Separate terminal 1, AbstractPollingIoProcessor$Processor warnings on quit
+
+* ``30-39_Subscriber-terminal.sh`` : Separate terminal 2
+
+### 40-00 Local Hints
+
+Reminder: Hints must be Done on the publisher before publishing to a subscriber - so the
+two collections match during the backup operation. If not an exception is thrown.
+
+* ``reset.sh`` : Reset the test/ directory
+
+* ``40-01_Hints-publisher.sh`` : Run once
+
+ * ``40-02_Hints-publisher-dryrun.sh`` : Run once
+
+ * ``40-22_Publisher-dryrun.sh`` : Run once, results & copies will be wrong because hints not processed
+
+ * ``40-23_Publisher-backup.sh`` : Run once
+
+### 50-00 Remote Hints
+
+* ``reset.sh`` : Reset the test/ directory
+
+* ``50-01_Hints-publisher.sh`` : Run once
+
+* ``50-21_Subscriber-One-listener.sh`` : Separate terminal 1
+
+* ``50-22_Publisher-One-dryrun.sh`` : Separate terminal 2, results & copies will be wrong because hints not processed
+
+* ``50-21_Subscriber-One-listener.sh`` : Separate terminal 1
+
+* ``50-23_Publisher-One-backup.sh`` : Separate terminal 2
+
+* ``50-31_Subscriber-Two-listener.sh`` : Separate terminal 1
+
+* ``50-32_Publisher-Two-dryrun.sh`` : Separate terminal 2
+
+* ``50-31_Subscriber-Two-listener.sh`` : Separate terminal 1, various "Does not exist" because of test setup
+
+* ``50-33_Publisher-Two-backup.sh`` : Separate terminal 2
+
+### 60-00 Local Hint Tracker
+
+* ``reset.sh`` : Reset the test/ directory
+
+* ``60-01_Hints-publisher.sh`` : Run once, hints are tracked locally
+
+* ``60-22_Publisher-One-dryrun.sh`` : Run once
+
+* ``60-23_Publisher-One-backup.sh`` : Run once
+
+* ``60-32_Publisher-Two-dryrun.sh`` : Run once
+
+* ``60-33_Publisher-Two-backup.sh`` : Run once
+
+* ``60-23_Publisher-One-backup.sh`` : Run once
+
+* ``60-33_Publisher-Two-backup.sh`` : Run once, all test/ directory .els files should be gone and the
+ test/hints/datastore/ directory should be empty
+
+### 70-00 Remote Hint Server
+
+* ``reset.sh`` : Reset the test/ directory
+
+* ``70-01_Hints-publisher.sh`` : Run once, hints are tracked locally
+
+* ``70-10_Status-Server-listener.sh`` : Separate terminal 1
+
+* ``70-21_Subscriber-One-listener-quit.sh`` : Separate terminal 2
+
+* ``70-22_Publisher-One-dryrun.sh`` ; Separate terminal 3, all processes should stop when done
+
+* ``70-10_Status-Server-listener.sh`` : Separate terminal 1
+
+* ``70-21_Subscriber-One-listener-quit.sh`` : Separate terminal 2
+
+* ``70-23_Publisher-One-backup.sh`` : Separate terminal 3, all processes should stop when done
+
+* ``70-10_Status-Server-listener.sh`` : Separate terminal 1
+
+* ``70-31_Subscriber-Two-listener.sh`` : Separate terminal 2
+
+* ``70-32_Publisher-Two-dryrun.sh`` : Separate terminal 3, status server continues to run
+
+* ``70-31_Subscriber-Two-listener.sh`` : Separate terminal 2
+
+* ``70-33_Publisher-Two-backup.sh`` : Separate terminal 3, status server continues to run
+
+* ``70-21_Subscriber-One-listener-quit.sh`` : Separate terminal 2
+
+* ``70-23_Publisher-One-backup.sh`` : Separate terminal 3, all processes should stop when done
+
+* ``70-10_Status-Server-listener.sh`` : Separate terminal 1
+
+* ``70-31_Subscriber-Two-listener.sh`` : Separate terminal 2
+
+* ``70-33_Publisher-Two-backup.sh`` : Separate terminal 3, status server continues to run,
+ all test/ directory .els files should be gone and the test/hints/datastore/ directory should be empty
+
+* ``70-90_Quit-Status-Server.sh`` : Separate terminal 2, stop the status server directly
+
diff --git a/mock/scripts/linux/00-00 Basic Functionality -------------------- b/mock/scripts/linux/00-00 Basic Functionality --------------------
new file mode 100644
index 00000000..0acc47c9
--- /dev/null
+++ b/mock/scripts/linux/00-00 Basic Functionality --------------------
@@ -0,0 +1 @@
+00-00
\ No newline at end of file
diff --git a/mock/scripts/linux/00-01_Version.sh b/mock/scripts/linux/00-01_Version.sh
new file mode 100755
index 00000000..f9406bbf
--- /dev/null
+++ b/mock/scripts/linux/00-01_Version.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --version
diff --git a/mock/scripts/linux/00-02_Bad-arguments.sh b/mock/scripts/linux/00-02_Bad-arguments.sh
new file mode 100755
index 00000000..4103bdbf
--- /dev/null
+++ b/mock/scripts/linux/00-02_Bad-arguments.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug -p test/publisher/publisher.json -s test/subscribe-one/subscriber-one.json -T test/subscriber-one/targets.json -F output/00-02_Bad-arguments.log -a-bad-argument
diff --git a/mock/scripts/linux/00-03_Validate.sh b/mock/scripts/linux/00-03_Validate.sh
new file mode 100755
index 00000000..dfccf3ab
--- /dev/null
+++ b/mock/scripts/linux/00-03_Validate.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug -p test/publisher/publisher.json -T test/subscriber-one/targets.json -F output/00-03_Validate.log --validate
diff --git a/mock/scripts/linux/00-04_Export.sh b/mock/scripts/linux/00-04_Export.sh
new file mode 100755
index 00000000..cc78c62e
--- /dev/null
+++ b/mock/scripts/linux/00-04_Export.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug -p test/publisher/publisher.json -T test/subscriber-one/targets.json -e output/00-04_Export.txt -i output/00-04_Export_collection.json -F output/00-04_Export.log
diff --git a/mock/scripts/linux/00-05_Duplicates.sh b/mock/scripts/linux/00-05_Duplicates.sh
new file mode 100755
index 00000000..a690179e
--- /dev/null
+++ b/mock/scripts/linux/00-05_Duplicates.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug -p test/publisher/publisher.json -T test/subscriber-one/targets.json -F output/00-05_Duplicates.log --duplicates
diff --git a/mock/scripts/linux/00-06_Duplicates-crosscheck.sh b/mock/scripts/linux/00-06_Duplicates-crosscheck.sh
new file mode 100755
index 00000000..48e5f6be
--- /dev/null
+++ b/mock/scripts/linux/00-06_Duplicates-crosscheck.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug -p test/publisher/publisher.json -T test/subscriber-one/targets.json -F output/00-06_Duplicates-crosscheck.log --duplicates --cross-check
diff --git a/mock/scripts/linux/10-00 Local Backup -------------------- b/mock/scripts/linux/10-00 Local Backup --------------------
new file mode 100644
index 00000000..d137b9cf
--- /dev/null
+++ b/mock/scripts/linux/10-00 Local Backup --------------------
@@ -0,0 +1 @@
+10-00
\ No newline at end of file
diff --git a/mock/scripts/linux/10-22_Backup-dryrun.sh b/mock/scripts/linux/10-22_Backup-dryrun.sh
new file mode 100755
index 00000000..fe7761fc
--- /dev/null
+++ b/mock/scripts/linux/10-22_Backup-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T test/subscriber-one/targets.json -m output/10-22_Backup-dryrun_mismatches.txt -W output/10-22_Backup-dryrun_whatsnew.txt -F output/10-22_Backup-dryrun.log --dry-run
diff --git a/mock/scripts/linux/10-23_Backup.sh b/mock/scripts/linux/10-23_Backup.sh
new file mode 100755
index 00000000..98e4fe72
--- /dev/null
+++ b/mock/scripts/linux/10-23_Backup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T test/subscriber-one/targets.json -m output/10-23_Backup_mismatches.txt -W output/10-23_Backup_whatsnew.txt -F output/10-23_Backup.log
diff --git a/mock/scripts/linux/10-24_Backup-exclude-lib.sh b/mock/scripts/linux/10-24_Backup-exclude-lib.sh
new file mode 100755
index 00000000..92015db7
--- /dev/null
+++ b/mock/scripts/linux/10-24_Backup-exclude-lib.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T test/subscriber-one/targets.json -m output/10-24_Backup-exclude-lib_mismatches.txt -W output/10-24_Backup-exclude-lib_whatsnew.txt -F output/10-24_Backup-exclude-lib.log -L "TV Shows"
diff --git a/mock/scripts/linux/10-25_Backup-include-lib.sh b/mock/scripts/linux/10-25_Backup-include-lib.sh
new file mode 100755
index 00000000..958eb223
--- /dev/null
+++ b/mock/scripts/linux/10-25_Backup-include-lib.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T test/subscriber-one/targets.json -m output/10-25_Backup-include-lib_mismatches.txt -W output/10-25_Backup-include-lib_whatsnew.txt -F output/10-25_Backup-include-lib.log -l Movies
diff --git a/mock/scripts/linux/20-00 Remote Backup -------------------- b/mock/scripts/linux/20-00 Remote Backup --------------------
new file mode 100644
index 00000000..f35bbeaa
--- /dev/null
+++ b/mock/scripts/linux/20-00 Remote Backup --------------------
@@ -0,0 +1 @@
+20-00
\ No newline at end of file
diff --git a/mock/scripts/linux/20-21_Subscriber-listener.sh b/mock/scripts/linux/20-21_Subscriber-listener.sh
new file mode 100755
index 00000000..f34fa884
--- /dev/null
+++ b/mock/scripts/linux/20-21_Subscriber-listener.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug --remote S -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -F output/20-21_Subscriber-listener.log
diff --git a/mock/scripts/linux/20-22_Publisher-dryrun.sh b/mock/scripts/linux/20-22_Publisher-dryrun.sh
new file mode 100755
index 00000000..fdf54735
--- /dev/null
+++ b/mock/scripts/linux/20-22_Publisher-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug --remote P -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -m output/20-22_Publisher-dryrun_mismatches.txt -W output/20-22_Publisher-dryrun_whatsnew.txt -F output/20-22_Publisher-dryrun.log --dry-run
diff --git a/mock/scripts/linux/20-23_Publisher-backup.sh b/mock/scripts/linux/20-23_Publisher-backup.sh
new file mode 100755
index 00000000..efbf42c1
--- /dev/null
+++ b/mock/scripts/linux/20-23_Publisher-backup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug --remote P -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T test/subscriber-one/targets.json -m output/20-23_Publisher-backup_mismatches.txt -W output/20-23_Publisher-backup_whatsnew.txt -F output/20-23_Publisher-backup.log
diff --git a/mock/scripts/linux/30-00 Interactive Terminals -------------------- b/mock/scripts/linux/30-00 Interactive Terminals --------------------
new file mode 100644
index 00000000..0eab92d7
--- /dev/null
+++ b/mock/scripts/linux/30-00 Interactive Terminals --------------------
@@ -0,0 +1 @@
+30-00
\ No newline at end of file
diff --git a/mock/scripts/linux/30-21_Subscriber-listener.sh b/mock/scripts/linux/30-21_Subscriber-listener.sh
new file mode 100755
index 00000000..81d6bc5e
--- /dev/null
+++ b/mock/scripts/linux/30-21_Subscriber-listener.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug --remote S --authorize sharkbait -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -F output/30-21_Subscriber-listener.log
diff --git a/mock/scripts/linux/30-29_Publisher-manual.sh b/mock/scripts/linux/30-29_Publisher-manual.sh
new file mode 100755
index 00000000..86c61d11
--- /dev/null
+++ b/mock/scripts/linux/30-29_Publisher-manual.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug --remote M -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -F output/30-29_Publisher-manual.log
diff --git a/mock/scripts/linux/30-31_Publisher-listener.sh b/mock/scripts/linux/30-31_Publisher-listener.sh
new file mode 100755
index 00000000..ac87fd1d
--- /dev/null
+++ b/mock/scripts/linux/30-31_Publisher-listener.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug --remote L --authorize sharkbait -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -F output/30-31_Publisher-listener.log
diff --git a/mock/scripts/linux/30-39_Subscriber-terminal.sh b/mock/scripts/linux/30-39_Subscriber-terminal.sh
new file mode 100755
index 00000000..70dc8dd6
--- /dev/null
+++ b/mock/scripts/linux/30-39_Subscriber-terminal.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -c debug -d debug --remote T -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -F output/30-39_Subscriber-terminal.log
diff --git a/mock/scripts/linux/40-00 Local Hints -------------------- b/mock/scripts/linux/40-00 Local Hints --------------------
new file mode 100644
index 00000000..b64d11b9
--- /dev/null
+++ b/mock/scripts/linux/40-00 Local Hints --------------------
@@ -0,0 +1 @@
+40-00
\ No newline at end of file
diff --git a/mock/scripts/linux/40-01_Hints-publisher.sh b/mock/scripts/linux/40-01_Hints-publisher.sh
new file mode 100755
index 00000000..0e0073b6
--- /dev/null
+++ b/mock/scripts/linux/40-01_Hints-publisher.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -K test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -T -F output/40-01_Hints-publisher.log
diff --git a/mock/scripts/linux/40-02_Hints-publisher-dryrun.sh b/mock/scripts/linux/40-02_Hints-publisher-dryrun.sh
new file mode 100755
index 00000000..01f6f6a9
--- /dev/null
+++ b/mock/scripts/linux/40-02_Hints-publisher-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -K test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -T -F output/40-02_Hints-publisher-dryrun.log --dry-run
diff --git a/mock/scripts/linux/40-22_Publisher-dryrun.sh b/mock/scripts/linux/40-22_Publisher-dryrun.sh
new file mode 100755
index 00000000..af4c2327
--- /dev/null
+++ b/mock/scripts/linux/40-22_Publisher-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -k test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -m output/40-22_Publisher-dryrun_mismatches.txt -W output/40-22_Publisher-dryrun_whatsnew.txt -F output/40-22_Publisher-dryrun.log --dry-run
diff --git a/mock/scripts/linux/40-23_Publisher-backup.sh b/mock/scripts/linux/40-23_Publisher-backup.sh
new file mode 100755
index 00000000..5be5ccab
--- /dev/null
+++ b/mock/scripts/linux/40-23_Publisher-backup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -k test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -m output/40-23_Publisher-backup_mismatches.txt -W output/40-23_Publisher-backup_whatsnew.txt -F output/40-23_Publisher-backup.log
diff --git a/mock/scripts/linux/50-00 Remote Hints -------------------- b/mock/scripts/linux/50-00 Remote Hints --------------------
new file mode 100644
index 00000000..244d4ad1
--- /dev/null
+++ b/mock/scripts/linux/50-00 Remote Hints --------------------
@@ -0,0 +1 @@
+50-00
\ No newline at end of file
diff --git a/mock/scripts/linux/50-01_Hints-publisher.sh b/mock/scripts/linux/50-01_Hints-publisher.sh
new file mode 100755
index 00000000..6f61ebb5
--- /dev/null
+++ b/mock/scripts/linux/50-01_Hints-publisher.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -K test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -T -F output/50-01_Hints-publisher.log
diff --git a/mock/scripts/linux/50-21_Subscriber-One-listener.sh b/mock/scripts/linux/50-21_Subscriber-One-listener.sh
new file mode 100755
index 00000000..36011d2b
--- /dev/null
+++ b/mock/scripts/linux/50-21_Subscriber-One-listener.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -k test/test-hints.keys -c debug -d debug --remote S --authorize sharkbait -p test/publisher/publisher.json -S test/subscriber-one/subscriber-one.json -T -F output/50-21_Subscriber-One-listener.log
diff --git a/mock/scripts/linux/50-22_Publisher-One-dryrun.sh b/mock/scripts/linux/50-22_Publisher-One-dryrun.sh
new file mode 100755
index 00000000..66a481df
--- /dev/null
+++ b/mock/scripts/linux/50-22_Publisher-One-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -k test/test-hints.keys -c debug -d debug --remote P -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -m output/50-22_Publisher-One-dryrun_mismatches.txt -W output/50-22_Publisher-One-dryrun_whatsnew.txt -F output/50-22_Publisher-One-dryrun.log --dry-run
diff --git a/mock/scripts/linux/50-23_Publisher-One-backup.sh b/mock/scripts/linux/50-23_Publisher-One-backup.sh
new file mode 100755
index 00000000..becf5785
--- /dev/null
+++ b/mock/scripts/linux/50-23_Publisher-One-backup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -k test/test-hints.keys -c debug -d debug --remote P -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -m output/50-23_Publisher-One-backup_mismatches.txt -W output/50-23_Publisher-One-backup_whatsnew.txt -F output/50-23_Publisher-One-backup.log
diff --git a/mock/scripts/linux/50-31_Subscriber-Two-listener.sh b/mock/scripts/linux/50-31_Subscriber-Two-listener.sh
new file mode 100755
index 00000000..32595b3f
--- /dev/null
+++ b/mock/scripts/linux/50-31_Subscriber-Two-listener.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -k test/test-hints.keys -c debug -d debug --remote S --authorize sharkbait -p test/publisher/publisher.json -S test/subscriber-two/subscriber-two.json -T -F output/50-31_Subscriber-Two-listener.log
diff --git a/mock/scripts/linux/50-32_Publisher-Two-dryrun.sh b/mock/scripts/linux/50-32_Publisher-Two-dryrun.sh
new file mode 100755
index 00000000..44f70d04
--- /dev/null
+++ b/mock/scripts/linux/50-32_Publisher-Two-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -k test/test-hints.keys -c debug -d debug --remote P -p test/publisher/publisher.json -S test/subscriber-two/subscriber-two.json -T -m output/50-32_Publisher-Two-dryrun_mismatches.txt -W output/50-32_Publisher-Two-dryrun_whatsnew.txt -F output/50-32_Publisher-Two-dryrun.log --dry-run
diff --git a/mock/scripts/linux/50-33_Publisher-Two-backup.sh b/mock/scripts/linux/50-33_Publisher-Two-backup.sh
new file mode 100755
index 00000000..7c0fdbbb
--- /dev/null
+++ b/mock/scripts/linux/50-33_Publisher-Two-backup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar -k test/test-hints.keys -c debug -d debug --remote P -p test/publisher/publisher.json -s test/subscriber-two/subscriber-two.json -T -m output/50-33_Publisher-Two-dryrun_mismatches.txt -W output/50-33_Publisher-Two-dryrun_whatsnew.txt -F output/50-33_Publisher-Two-dryrun.log
diff --git a/mock/scripts/linux/60-00 Local Hint Tracker -------------------- b/mock/scripts/linux/60-00 Local Hint Tracker --------------------
new file mode 100644
index 00000000..60e43ff4
--- /dev/null
+++ b/mock/scripts/linux/60-00 Local Hint Tracker --------------------
@@ -0,0 +1 @@
+60-00
\ No newline at end of file
diff --git a/mock/scripts/linux/60-01_Hints-publisher.sh b/mock/scripts/linux/60-01_Hints-publisher.sh
new file mode 100755
index 00000000..5108aaeb
--- /dev/null
+++ b/mock/scripts/linux/60-01_Hints-publisher.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -K test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -T -F output/60-01_Hints-publisher.log
diff --git a/mock/scripts/linux/60-22_Publisher-One-dryrun.sh b/mock/scripts/linux/60-22_Publisher-One-dryrun.sh
new file mode 100755
index 00000000..69ca6891
--- /dev/null
+++ b/mock/scripts/linux/60-22_Publisher-One-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -m output/60-22_Publisher-One-dryrun_mismatches.txt -W output/60-22_Publisher-One-dryrun_whatsnew.txt -F output/60-22_Publisher-One-dryrun.log --dry-run
diff --git a/mock/scripts/linux/60-23_Publisher-One-backup.sh b/mock/scripts/linux/60-23_Publisher-One-backup.sh
new file mode 100755
index 00000000..281f2fec
--- /dev/null
+++ b/mock/scripts/linux/60-23_Publisher-One-backup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -m output/60-23_Publisher-One-backup_mismatches.txt -W output/60-23_Publisher-One-backup_whatsnew.txt -F output/60-23_Publisher-One-backup.log
diff --git a/mock/scripts/linux/60-32_Publisher-Two-dryrun.sh b/mock/scripts/linux/60-32_Publisher-Two-dryrun.sh
new file mode 100755
index 00000000..73f3a726
--- /dev/null
+++ b/mock/scripts/linux/60-32_Publisher-Two-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-two/subscriber-two.json -T -m output/60-32_Publisher-Two-dryrun_mismatches.txt -W output/60-32_Publisher-Two-dryrun_whatsnew.txt -F output/60-32_Publisher-Two-dryrun.log --dry-run
diff --git a/mock/scripts/linux/60-33_Publisher-Two-backup.sh b/mock/scripts/linux/60-33_Publisher-Two-backup.sh
new file mode 100755
index 00000000..b1f02496
--- /dev/null
+++ b/mock/scripts/linux/60-33_Publisher-Two-backup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -s test/subscriber-two/subscriber-two.json -T -m output/60-33_Publisher-Two-backup_mismatches.txt -W output/60-33_Publisher-Two-backup_whatsnew.txt -F output/60-33_Publisher-Two-backup.log
diff --git a/mock/scripts/linux/70-00 Remote Hint Server -------------------- b/mock/scripts/linux/70-00 Remote Hint Server --------------------
new file mode 100644
index 00000000..58634ed2
--- /dev/null
+++ b/mock/scripts/linux/70-00 Remote Hint Server --------------------
@@ -0,0 +1 @@
+70-00
\ No newline at end of file
diff --git a/mock/scripts/linux/70-01_Hints-publisher.sh b/mock/scripts/linux/70-01_Hints-publisher.sh
new file mode 100755
index 00000000..35a8b668
--- /dev/null
+++ b/mock/scripts/linux/70-01_Hints-publisher.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -K test/test-hints.keys -c debug -d debug -p test/publisher/publisher.json -T -F output/70-01_Hints-publisher.log
diff --git a/mock/scripts/linux/70-10_Status-Server-listener.sh b/mock/scripts/linux/70-10_Status-Server-listener.sh
new file mode 100755
index 00000000..a211eee6
--- /dev/null
+++ b/mock/scripts/linux/70-10_Status-Server-listener.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hint-server test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug -F output/70-10_Status-Server-listener.log
diff --git a/mock/scripts/linux/70-21_Subscriber-One-listener-quit.sh b/mock/scripts/linux/70-21_Subscriber-One-listener-quit.sh
new file mode 100755
index 00000000..fb2a4768
--- /dev/null
+++ b/mock/scripts/linux/70-21_Subscriber-One-listener-quit.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --quit-status --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug --remote S --authorize sharkbait -p test/publisher/publisher.json -S test/subscriber-one/subscriber-one.json -T -F output/70-21_Subscriber-One-listener.log
diff --git a/mock/scripts/linux/70-22_Publisher-One-dryrun.sh b/mock/scripts/linux/70-22_Publisher-One-dryrun.sh
new file mode 100755
index 00000000..83045b51
--- /dev/null
+++ b/mock/scripts/linux/70-22_Publisher-One-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug --remote P -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -m output/70-22_Publisher-One-dryrun_mismatches.txt -W output/70-22_Publisher-One-dryrun_whatsnew.txt -F output/70-22_Publisher-One-dryrun.log --dry-run
diff --git a/mock/scripts/linux/70-23_Publisher-One-backup.sh b/mock/scripts/linux/70-23_Publisher-One-backup.sh
new file mode 100755
index 00000000..9040eee7
--- /dev/null
+++ b/mock/scripts/linux/70-23_Publisher-One-backup.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug --remote P -p test/publisher/publisher.json -s test/subscriber-one/subscriber-one.json -T -m output/70-23_Publisher-One-backup_mismatches.txt -W output/70-23_Publisher-One-backup_whatsnew.txt -F output/70-23_Publisher-One-backup.log
+
diff --git a/mock/scripts/linux/70-31_Subscriber-Two-listener.sh b/mock/scripts/linux/70-31_Subscriber-Two-listener.sh
new file mode 100755
index 00000000..23ee8d30
--- /dev/null
+++ b/mock/scripts/linux/70-31_Subscriber-Two-listener.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug --remote S --authorize sharkbait -p test/publisher/publisher.json -S test/subscriber-two/subscriber-two.json -T -F output/70-31_Subscriber-Two-listener.log
diff --git a/mock/scripts/linux/70-32_Publisher-Two-dryrun.sh b/mock/scripts/linux/70-32_Publisher-Two-dryrun.sh
new file mode 100755
index 00000000..73fe884b
--- /dev/null
+++ b/mock/scripts/linux/70-32_Publisher-Two-dryrun.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug --remote P -p test/publisher/publisher.json -s test/subscriber-two/subscriber-two.json -T -m output/70-32_Publisher-Two-dryrun_mismatches.txt -W output/70-32_Publisher-Two-dryrun_whatsnew.txt -F output/70-32_Publisher-Two-dryrun.log --dry-run
diff --git a/mock/scripts/linux/70-33_Publisher-Two-backup.sh b/mock/scripts/linux/70-33_Publisher-Two-backup.sh
new file mode 100755
index 00000000..30be35e4
--- /dev/null
+++ b/mock/scripts/linux/70-33_Publisher-Two-backup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --hints test/hints/hint-server.json -k test/test-hints.keys -c debug -d debug --remote P -p test/publisher/publisher.json -S test/subscriber-two/subscriber-two.json -T -m output/70-33_Publisher-Two-backup_mismatches.txt -W output/70-33_Publisher-Two-bacup_whatsnew.txt -F output/70-33_Publisher-Two-backup.log
diff --git a/mock/scripts/linux/70-90_Quit-Status-Server.sh b/mock/scripts/linux/70-90_Quit-Status-Server.sh
new file mode 100755
index 00000000..179e5a77
--- /dev/null
+++ b/mock/scripts/linux/70-90_Quit-Status-Server.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+base=`dirname $0`
+if [ "$base" = "." ]; then
+ base=$PWD
+fi
+cd "$base"
+
+name=`basename $0 .sh`
+
+cd ../..
+
+
+if [ ! -d output ]; then
+ mkdir output
+fi
+
+java -jar ../deploy/ELS.jar --force-quit --hints test/hints/hint-server.json -p test/publisher/publisher.json -c debug -d debug -F output/70-99_Quit-Status-Server.log
diff --git a/mock/scripts/linux/clear-output.sh b/mock/scripts/linux/clear-output.sh
new file mode 100755
index 00000000..9b7a3cb8
--- /dev/null
+++ b/mock/scripts/linux/clear-output.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+base=`dirname $0`
+cd ${base}
+
+if [ "$1" != "-f" ]; then
+ echo ""
+ echo "Clear output directory"
+ read -p "Confirm: DESTROY the logs in the output directory (Y/n)? " R
+ R=${R:0:1}
+ if [ "$R" != 'y' -a "$R" != 'Y' ]; then
+ echo -e "Cancelled\n"
+ exit 1
+ fi
+fi
+
+if [ -d ../../output ]; then
+ rm -f ../../output/*
+fi
+
+echo ""
+echo -e "Clear done"
+date
+echo ""
+
diff --git a/mock/reset.sh b/mock/scripts/linux/reset.sh
similarity index 50%
rename from mock/reset.sh
rename to mock/scripts/linux/reset.sh
index b59abb60..f5123147 100755
--- a/mock/reset.sh
+++ b/mock/scripts/linux/reset.sh
@@ -1,24 +1,29 @@
-#!/bin/bash
-
-base=`dirname $0`
-cd ${base}
-
-if [ "$1" != "-f" ]; then
- if [ -e ./TestRun ]; then
- echo ""
- echo "Reset TestRun Directory"
- read -p "Confirm: DESTROY TestRun directory and recreate from templates (y/N)? " R
- R=${R:0:1}
- if [ "$R" != 'y' -a "$R" != 'Y' ]; then
- echo -e "Cancelled\n"
- exit 1
- fi
- fi
-fi
-
-rm -rf ./TestRun
-rm -f ./logmunger.log
-
-cp -rpv ./Template_Copy-Only ./TestRun
-
-echo -e "Done\n"
+#!/bin/bash
+
+base=`dirname $0`
+cd ${base}
+cd ../..
+
+if [ "$1" != "-f" ]; then
+ if [ -e ./TestRun ]; then
+ echo ""
+ echo "Reset Test Directory"
+ read -p "Confirm: DESTROY Test directory and recreate from templates (y/N)? " R
+ R=${R:0:1}
+ if [ "$R" != 'y' -a "$R" != 'Y' ]; then
+ echo -e "Cancelled\n"
+ exit 1
+ fi
+ fi
+fi
+
+rm -rf ./test
+rm -f ./*.log
+
+cp -rpv ./media-base_copy-only ./test
+echo ""
+
+echo -e "Reset done"
+date
+echo ""
+
diff --git a/src/com/groksoft/els/Configuration.java b/src/com/groksoft/els/Configuration.java
index 0076dbce..304ea6f2 100755
--- a/src/com/groksoft/els/Configuration.java
+++ b/src/com/groksoft/els/Configuration.java
@@ -8,7 +8,7 @@
import java.util.ArrayList;
/**
- * Configuration
+ * Configuration class.
*
* Contains all command-line options and any other application-level configuration.
*/
@@ -22,9 +22,11 @@ public class Configuration
public static final int RENAME_DIRECTORIES = 2;
public static final int RENAME_FILES = 1;
public static final int RENAME_NONE = 0;
+ public static final int STATUS_SERVER = 6;
+ public static final int STATUS_SERVER_FORCE_QUIT = 7;
public static final int SUBSCRIBER_LISTENER = 2;
public static final int SUBSCRIBER_TERMINAL = 5;
- private final String PROGRAM_VERSION = "3.0.0";
+ private final String PROGRAM_VERSION = "3.1.0";
private final String PROGRAM_NAME = "ELS : Entertainment Library Synchronizer";
private String authorizedPassword = "";
private String consoleLevel = "debug"; // Levels: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, and OFF
@@ -38,6 +40,7 @@ public class Configuration
private boolean forceTargets = false;
private String hintKeysFile = "";
private boolean hintSkipMainProcess = false;
+ private String hintsDaemonFilename = "";
private String logFilename = "";
private boolean logOverwrite = false;
private String mismatchFilename = "";
@@ -47,6 +50,7 @@ public class Configuration
private boolean publishOperation = true;
private String publisherCollectionFilename = "";
private String publisherLibrariesFileName = "";
+ private boolean quitStatusServer = false;
private int remoteFlag = NOT_REMOTE;
private String remoteType = "-";
private boolean renaming = false;
@@ -57,6 +61,7 @@ public class Configuration
private ArrayList selectedLibraryNames = new ArrayList<>();
private boolean specificExclude = false;
private boolean specificLibrary = false;
+ private String statusTrackerFilename = "";
private String subscriberCollectionFilename = "";
private String subscriberLibrariesFileName = "";
private boolean targetsEnabled = false;
@@ -120,6 +125,14 @@ public void dump()
logger.info(SHORT, " cfg: -e Export text filename = " + getExportTextFilename());
}
logger.info(SHORT, " cfg: -f Log filename = " + getLogFilename());
+ if (statusTrackerFilename != null && statusTrackerFilename.length() > 0)
+ {
+ logger.info(SHORT, " cfg: -h Hint status server: " + getStatusTrackerFilename());
+ }
+ if (hintsDaemonFilename != null && hintsDaemonFilename.length() > 0)
+ {
+ logger.info(SHORT, " cfg: -H Hints status server daemon: " + getHintsDaemonFilename());
+ }
if (getExportCollectionFilename().length() > 0)
{
logger.info(SHORT, " cfg: -i Export collection JSON filename = " + getExportCollectionFilename());
@@ -158,6 +171,10 @@ public void dump()
{
logger.info(SHORT, " cfg: -P Publisher Collection filename = " + getPublisherCollectionFilename());
}
+ if (isQuitStatusServer())
+ {
+ logger.info(SHORT, " cfg: -q Status server QUIT");
+ }
logger.info(SHORT, " cfg: -r Remote session type = " + getRemoteType());
if (getSubscriberLibrariesFileName().length() > 0)
{
@@ -290,11 +307,36 @@ public void setExportTextFilename(String exportTextFilename)
this.exportTextFilename = exportTextFilename;
}
+ /**
+ * Gets Hint Keys filename
+ *
+ * @return String filename
+ */
public String getHintKeysFile()
{
return hintKeysFile;
}
+ /**
+ * Gets Hint Status Server filename
+ *
+ * @return String filename
+ */
+ public String getHintsDaemonFilename()
+ {
+ return hintsDaemonFilename;
+ }
+
+ /**
+ * Sets the Hint Status Server filename
+ *
+ * @param hintsDaemonFilename
+ */
+ public void setHintsDaemonFilename(String hintsDaemonFilename)
+ {
+ this.hintsDaemonFilename = hintsDaemonFilename;
+ }
+
/**
* Gets log filename
*
@@ -358,7 +400,7 @@ public String getPattern()
*
* @return the Main version
*/
- public String getProgramVersionN()
+ public String getProgramVersion()
{
return PROGRAM_VERSION;
}
@@ -406,7 +448,7 @@ public void setPublisherLibrariesFileName(String publisherLibrariesFileName)
/**
* Gets remote flag
*
- * @return the remote flag, 0 = none, 1 = publisher, 2 = subscriber, 3 = pub terminal, 4 = pub listener, 5 = sub terminal
+ * @return the remote flag, 0 = none, 1 = publisher, 2 = subscriber, 3 = pub terminal, 4 = pub listener, 5 = sub terminal, 6 = status server, 7 = force quit status server
*/
public int getRemoteFlag()
{
@@ -489,6 +531,26 @@ public ArrayList getSelectedLibraryNames()
return selectedLibraryNames;
}
+ /**
+ * Get the Hint Status Tracker configuration filename
+ *
+ * @return String filename
+ */
+ public String getStatusTrackerFilename()
+ {
+ return statusTrackerFilename;
+ }
+
+ /**
+ * Set the Hint Status Tracker configuration filename
+ *
+ * @param statusTrackerFilename
+ */
+ public void setStatusTrackerFilename(String statusTrackerFilename)
+ {
+ this.statusTrackerFilename = statusTrackerFilename;
+ }
+
/**
* Gets subscriber import filename
*
@@ -569,11 +631,21 @@ public void setWhatsNewFilename(String whatsNewFilename)
this.whatsNewFilename = whatsNewFilename;
}
+ /**
+ * Is a duplicates cross-check enabled?
+ *
+ * @return true if enabled, else false
+ */
public boolean isCrossCheck()
{
return crossCheck;
}
+ /**
+ * Sets duplicates cross-check
+ *
+ * @param crossCheck
+ */
public void setCrossCheck(boolean crossCheck)
{
this.crossCheck = crossCheck;
@@ -599,11 +671,21 @@ public void setDryRun(boolean dryRun)
this.dryRun = dryRun;
}
+ /**
+ * Are duplicates being checked?
+ *
+ * @return true if duplcates checking is enabled, else false
+ */
public boolean isDuplicateCheck()
{
return duplicateCheck;
}
+ /**
+ * Sets duplcates checking
+ *
+ * @param duplicateCheck
+ */
public void setDuplicateCheck(boolean duplicateCheck)
{
this.duplicateCheck = duplicateCheck;
@@ -666,31 +748,62 @@ public void setForceTargets(boolean forceTargets)
this.forceTargets = forceTargets;
}
+ /**
+ * Are only Hints being processed so skip the main munge process?
+ *
+ * @return true if hints skipping main process enabled
+ */
public boolean isHintSkipMainProcess()
{
return hintSkipMainProcess;
}
+ /**
+ * Sets if the hints option to skip the main munge process is enabled
+ *
+ * @param hintSkipMainProcess
+ */
public void setHintSkipMainProcess(boolean hintSkipMainProcess)
{
this.hintSkipMainProcess = hintSkipMainProcess;
}
+ /**
+ * Is the log to be overwritten?
+ *
+ * @return true to overwrite
+ */
public boolean isLogOverwrite()
{
return logOverwrite;
}
+ /**
+ * Sets if the log should be overwritten when the process starts
+ *
+ * @param logOverwrite
+ */
public void setLogOverwrite(boolean logOverwrite)
{
this.logOverwrite = logOverwrite;
}
+ /**
+ * Is the no back-fill option enabled so the default behavior of filling-in
+ * original locations with new files is disabled?
+ *
+ * @return true if no back-fill is enabled
+ */
public boolean isNoBackFill()
{
return noBackFill;
}
+ /**
+ * Sets the no back-fill option
+ *
+ * @param noBackFill
+ */
public void setNoBackFill(boolean noBackFill)
{
this.noBackFill = noBackFill;
@@ -754,6 +867,26 @@ public boolean isPublisherTerminal()
return (getRemoteFlag() == PUBLISHER_MANUAL);
}
+ /**
+ * Should the current process command the Hint Status Server to quit?
+ *
+ * @return true if the command is to be sent
+ */
+ public boolean isQuitStatusServer()
+ {
+ return quitStatusServer;
+ }
+
+ /**
+ * Sets whether this process should command the Hint Status Server to quit
+ *
+ * @param quitStatusServer
+ */
+ public void setQuitStatusServer(boolean quitStatusServer)
+ {
+ this.quitStatusServer = quitStatusServer;
+ }
+
/**
* Returns true if this is a publisher process, automatically execute the process
*
@@ -887,6 +1020,16 @@ public void setSpecificLibrary(boolean sense)
this.specificLibrary = sense;
}
+ /**
+ * Returns true if this is a hint status server
+ *
+ * @return true/false
+ */
+ public boolean isStatusServer()
+ {
+ return (getRemoteFlag() == STATUS_SERVER);
+ }
+
/**
* Returns true if subscriber is in listener mode
*/
@@ -919,11 +1062,31 @@ public void setTargetsEnabled(boolean sense)
targetsEnabled = sense;
}
+ /**
+ * Is a Hint Status Tracker being used?
+ *
+ * @return true if so
+ */
+ public boolean isUsingHintTracker()
+ {
+ return (statusTrackerFilename.length() > 0);
+ }
+
+ /**
+ * Is the validation of collections and targets enabled?
+ *
+ * @return true if validation should be done
+ */
public boolean isValidation()
{
return validation;
}
+ /**
+ * Sets the collection and targets validation option
+ *
+ * @param validation
+ */
public void setValidation(boolean validation)
{
this.validation = validation;
@@ -1041,6 +1204,33 @@ public void parseCommandLine(String[] args) throws MungeException
throw new MungeException("Error: -f requires a log filename");
}
break;
+ case "-h": // hint status tracker, v3.1.0
+ case "--hints":
+ if (index <= args.length - 2)
+ {
+ setStatusTrackerFilename(args[index + 1]);
+ ++index;
+ setPublishOperation(false);
+ }
+ else
+ {
+ throw new MungeException("Error: -h requires a hint status server repository filename");
+ }
+ break;
+ case "-H": // hint status server, v3.1.0
+ case "--hint-server":
+ this.remoteFlag = STATUS_SERVER;
+ if (index <= args.length - 2)
+ {
+ setHintsDaemonFilename(args[index + 1]);
+ ++index;
+ setPublishOperation(false);
+ }
+ else
+ {
+ throw new MungeException("Error: -H requires a hint status server repository filename");
+ }
+ break;
case "-i": // export publisher items to collection file
case "--export-items":
if (index <= args.length - 2)
@@ -1117,7 +1307,7 @@ public void parseCommandLine(String[] args) throws MungeException
throw new MungeException("Error: -m requires a mismatches output filename");
}
break;
- case "-n": // perform renaming
+ case "-n": // perform renaming
case "--rename":
setRenaming(true);
if (index <= args.length - 2)
@@ -1130,11 +1320,11 @@ public void parseCommandLine(String[] args) throws MungeException
throw new MungeException("Error: -n requires the type F | D | B");
}
break;
- case "-o":
+ case "-o": // overwrite
case "--overwrite":
setOverwrite(true);
break;
- case "-p": // publisher JSON libraries file
+ case "-p": // publisher JSON libraries file
case "--publisher-libraries":
if (index <= args.length - 2)
{
@@ -1158,6 +1348,15 @@ public void parseCommandLine(String[] args) throws MungeException
throw new MungeException("Error: -P requires a publisher collection filename");
}
break;
+ case "-q": // tell status server to quit
+ case "--quit-status":
+ setQuitStatusServer(true);
+ break;
+ case "-Q":
+ case "--force-quit":
+ setQuitStatusServer(true);
+ this.remoteFlag = STATUS_SERVER_FORCE_QUIT;
+ break;
case "-r": // remote session
case "--remote":
if (index <= args.length - 2)
@@ -1228,7 +1427,6 @@ public void parseCommandLine(String[] args) throws MungeException
case "--validate":
setValidation(true);
break;
- case "-h":
case "--version": // version
System.out.println("");
System.out.println(PROGRAM_NAME + ", Version " + PROGRAM_VERSION);
diff --git a/src/com/groksoft/els/Context.java b/src/com/groksoft/els/Context.java
index bc01a3ab..843759b6 100644
--- a/src/com/groksoft/els/Context.java
+++ b/src/com/groksoft/els/Context.java
@@ -1,23 +1,29 @@
package com.groksoft.els;
+import com.groksoft.els.repository.HintKeys;
import com.groksoft.els.repository.Repository;
import com.groksoft.els.sftp.ClientSftp;
import com.groksoft.els.sftp.ServeSftp;
import com.groksoft.els.stty.ClientStty;
import com.groksoft.els.stty.ServeStty;
+import com.groksoft.els.stty.hintServer.Datastore;
/**
- * Class to make passing these data easier
+ * Context class to make passing these data easier.
*/
public class Context
{
+ // some of these will be null at runtime depending on configuration
public ClientSftp clientSftp;
public ClientStty clientStty;
+ public Datastore datastore;
+ public HintKeys hintKeys;
+ public boolean hintMode = false;
public Repository publisherRepo;
public ServeSftp serveSftp;
public ServeStty serveStty;
+ public Repository statusRepo;
+ public ClientStty statusStty;
public Repository subscriberRepo;
public Transfer transfer;
- public boolean hintMode = false;
- public int type;
}
diff --git a/src/com/groksoft/els/Main.java b/src/com/groksoft/els/Main.java
index 306fbea1..26fe9081 100644
--- a/src/com/groksoft/els/Main.java
+++ b/src/com/groksoft/els/Main.java
@@ -1,10 +1,12 @@
package com.groksoft.els;
+import com.groksoft.els.repository.HintKeys;
import com.groksoft.els.repository.Repository;
import com.groksoft.els.sftp.ClientSftp;
import com.groksoft.els.sftp.ServeSftp;
import com.groksoft.els.stty.ClientStty;
import com.groksoft.els.stty.ServeStty;
+import com.groksoft.els.stty.hintServer.Datastore;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -16,12 +18,15 @@
import static com.groksoft.els.Configuration.*;
/**
- * ELS main program
+ * ELS main program.
*/
public class Main
{
+ private static Main els;
public boolean isListening = false;
- Context context = new Context();
+ boolean fault = false;
+ private Configuration cfg;
+ private Context context = new Context();
private Logger logger = null;
/**
@@ -38,34 +43,68 @@ public Main()
*/
public static void main(String[] args)
{
- Main els = new Main();
+ els = new Main();
els.process(args); // ELS Processor
} // main
/**
- * execute the process
+ * Connect to or setup hint tracking, connect to hint server if specified
+ *
+ * @param repo The Repository that is connecting to the tracker/server
+ * @throws Exception Configuration and connection exceptions
+ */
+ private void connectHintServer(Repository repo) throws Exception
+ {
+ if (cfg.isUsingHintTracker())
+ {
+ context.statusRepo = new Repository(cfg);
+ context.statusRepo.read(cfg.getStatusTrackerFilename());
+
+ if (cfg.isRemoteSession())
+ {
+ // start the serveStty client to the hints status server
+ context.statusStty = new ClientStty(cfg, false, true);
+ if (!context.statusStty.connect(repo, context.statusRepo))
+ {
+ throw new MungeException("Hint Status Server failed to connect");
+ }
+ String response = context.statusStty.receive(); // check the initial prompt
+ if (!response.startsWith("CMD"))
+ throw new MungeException("Bad initial response from status server: " + context.statusRepo.getLibraryData().libraries.description);
+ }
+ else
+ {
+ // Setup the hint status store, single instance
+ context.datastore = new Datastore(cfg, context);
+ context.datastore.initialize();
+ }
+ }
+ }
+
+ /**
+ * Execute the process
*
* @param args the input arguments
- * @return
+ * @return Return status
*/
public int process(String[] args)
{
int returnValue = 0;
ThreadGroup sessionThreads = null;
- Configuration cfg = new Configuration();
+ cfg = new Configuration();
Process proc;
Date stamp = new Date();
try
{
- MungeException ce = null;
+ MungeException cfgException = null;
try
{
cfg.parseCommandLine(args);
}
catch (MungeException e)
{
- ce = e; // configuration exception
+ cfgException = e; // configuration exception
}
// setup the logger based on configuration and/or defaults
@@ -92,16 +131,18 @@ public int process(String[] args)
// get the named logger
logger = LogManager.getLogger("applog");
- if (ce != null) // re-throw any configuration exception
- throw ce;
+ if (cfgException != null) // re-throw any configuration exception
+ throw cfgException;
+ //
// an execution of this program can only be configured as one of these
+ //
logger.info("+------------------------------------------");
switch (cfg.getRemoteFlag())
{
// handle standard local execution, no -r option
case NOT_REMOTE:
- logger.info("ELS Local Process begin, version " + cfg.getProgramVersionN());
+ logger.info("ELS Local Process begin, version " + cfg.getProgramVersion());
cfg.dump();
context.publisherRepo = readRepo(cfg, Repository.PUBLISHER, Repository.VALIDATE);
@@ -116,14 +157,17 @@ else if (cfg.isTargetsEnabled())
context.subscriberRepo = context.publisherRepo; // v3.00 for publisher ELS Hints
}
+ // setup the hint status server for local use if defined
+ connectHintServer(context.publisherRepo);
+
// the Process class handles the ELS process
proc = new Process(cfg, context);
- returnValue = proc.process();
+ fault = proc.process();
break;
// handle -r L publisher listener for remote subscriber -r T connections
case PUBLISHER_LISTENER:
- logger.info("ELS Publisher Listener begin, version " + cfg.getProgramVersionN());
+ logger.info("ELS Publisher Listener begin, version " + cfg.getProgramVersion());
cfg.dump();
context.publisherRepo = readRepo(cfg, Repository.PUBLISHER, Repository.VALIDATE);
@@ -132,6 +176,9 @@ else if (cfg.isTargetsEnabled())
// start servers for -r T & clients for get command in stty.publisher.Daemon
if (context.publisherRepo.isInitialized() && context.subscriberRepo.isInitialized())
{
+ // connect to the hint status server if defined
+ connectHintServer(context.publisherRepo);
+
// start serveStty server
sessionThreads = new ThreadGroup("PServer");
context.serveStty = new ServeStty(sessionThreads, 10, cfg, context, true);
@@ -150,7 +197,7 @@ else if (cfg.isTargetsEnabled())
// handle -r M publisher manual terminal to remote subscriber -r S
case PUBLISHER_MANUAL:
- logger.info("ELS Publisher Manual Terminal begin, version " + cfg.getProgramVersionN());
+ logger.info("ELS Publisher Manual Terminal begin, version " + cfg.getProgramVersion());
cfg.dump();
context.publisherRepo = readRepo(cfg, Repository.PUBLISHER, Repository.VALIDATE);
@@ -159,6 +206,9 @@ else if (cfg.isTargetsEnabled())
// start clients
if (context.publisherRepo.isInitialized() && context.subscriberRepo.isInitialized())
{
+ // connect to the hint status server if defined
+ connectHintServer(context.publisherRepo);
+
// start the serveStty client interactively
context.clientStty = new ClientStty(cfg, true, true);
if (context.clientStty.connect(context.publisherRepo, context.subscriberRepo))
@@ -182,7 +232,7 @@ else if (cfg.isTargetsEnabled())
// handle -r P execute the automated process to remote subscriber -r S
case PUBLISH_REMOTE:
- logger.info("ELS Publish Process to Remote Subscriber begin, version " + cfg.getProgramVersionN());
+ logger.info("ELS Publish Process to Remote Subscriber begin, version " + cfg.getProgramVersion());
cfg.dump();
context.publisherRepo = readRepo(cfg, Repository.PUBLISHER, Repository.VALIDATE);
@@ -191,6 +241,9 @@ else if (cfg.isTargetsEnabled())
// start clients
if (context.publisherRepo.isInitialized() && context.subscriberRepo.isInitialized())
{
+ // connect to the hint status server if defined
+ connectHintServer(context.publisherRepo);
+
// start the serveStty client for automation
context.clientStty = new ClientStty(cfg, false, true);
if (!context.clientStty.connect(context.publisherRepo, context.subscriberRepo))
@@ -207,7 +260,7 @@ else if (cfg.isTargetsEnabled())
// the Process class handles the ELS process
proc = new Process(cfg, context);
- returnValue = proc.process();
+ fault = proc.process();
}
else
{
@@ -217,7 +270,7 @@ else if (cfg.isTargetsEnabled())
// handle -r S subscriber listener for publisher -r P|M connections
case SUBSCRIBER_LISTENER:
- logger.info("ELS Subscriber Listener begin, version " + cfg.getProgramVersionN());
+ logger.info("ELS Subscriber Listener begin, version " + cfg.getProgramVersion());
cfg.dump();
if (!cfg.isTargetsEnabled())
@@ -229,6 +282,9 @@ else if (cfg.isTargetsEnabled())
// start servers
if (context.subscriberRepo.isInitialized() && context.publisherRepo.isInitialized())
{
+ // connect to the hint status server if defined
+ connectHintServer(context.subscriberRepo);
+
// start serveStty server
sessionThreads = new ThreadGroup("SServer");
context.serveStty = new ServeStty(sessionThreads, 10, cfg, context, true);
@@ -247,7 +303,7 @@ else if (cfg.isTargetsEnabled())
// handle -r T subscriber manual terminal to publisher -r L
case SUBSCRIBER_TERMINAL:
- logger.info("ELS Subscriber Manual Terminal begin, version " + cfg.getProgramVersionN());
+ logger.info("ELS Subscriber Manual Terminal begin, version " + cfg.getProgramVersion());
cfg.dump();
if (!cfg.isTargetsEnabled())
@@ -259,6 +315,9 @@ else if (cfg.isTargetsEnabled())
// start clients & servers for -r L for get command
if (context.subscriberRepo.isInitialized() && context.publisherRepo.isInitialized())
{
+ // connect to the hint status server if defined
+ connectHintServer(context.subscriberRepo);
+
// start the serveStty client interactively
context.clientStty = new ClientStty(cfg, true, true);
if (context.clientStty.connect(context.subscriberRepo, context.publisherRepo))
@@ -294,6 +353,67 @@ else if (cfg.isTargetsEnabled())
}
break;
+ // handle -H | --hint-server stand-alone hints status server
+ case STATUS_SERVER:
+ logger.info("ELS Hint Status Server begin, version " + cfg.getProgramVersion());
+ cfg.dump();
+
+ if (cfg.getHintKeysFile() == null || cfg.getHintKeysFile().length() == 0)
+ throw new MungeException("-H | --status-server requires a -k | -K hint keys file");
+
+ if (cfg.getPublisherLibrariesFileName().length() > 0 || cfg.getPublisherCollectionFilename().length() > 0)
+ throw new MungeException("-H | --status-server does not use -p | -P");
+
+ if (cfg.getSubscriberLibrariesFileName().length() > 0 || cfg.getSubscriberCollectionFilename().length() > 0)
+ throw new MungeException("-H | --status-server does not use -s | -S");
+
+ if (cfg.isTargetsEnabled())
+ throw new MungeException("-H | --status-server does not use targets");
+
+ // Get the hint status server repo
+ context.statusRepo = new Repository(cfg);
+ context.statusRepo.read(cfg.getHintsDaemonFilename());
+
+ // Get ELS hints keys if specified
+ context.hintKeys = new HintKeys(cfg, context);
+ context.hintKeys.read(cfg.getHintKeysFile());
+
+ // Setup the hint status store, single instance
+ context.datastore = new Datastore(cfg, context);
+ context.datastore.initialize();
+
+ // start server
+ if (context.statusRepo.isInitialized())
+ {
+ // start serveStty server
+ sessionThreads = new ThreadGroup("SServer");
+ context.serveStty = new ServeStty(sessionThreads, 10, cfg, context, true);
+ context.serveStty.startListening(context.statusRepo);
+ isListening = true;
+ }
+ else
+ {
+ throw new MungeException("Error initializing from hint status server JSON file");
+ }
+ break;
+
+ // handle -Q | --force-quit the hint status server remotely
+ case STATUS_SERVER_FORCE_QUIT:
+ logger.info("ELS Quit Hint Status Server begin, version " + cfg.getProgramVersion());
+ cfg.dump();
+
+ if (cfg.getStatusTrackerFilename() == null || cfg.getStatusTrackerFilename().length() == 0)
+ throw new MungeException("-Q | --force-quit requires a -h | --hints hint server JSON file");
+
+ context.publisherRepo = readRepo(cfg, Repository.PUBLISHER, Repository.VALIDATE);
+
+ connectHintServer(context.publisherRepo);
+
+ // force the cfg setting & let this process end normally
+ // that will send the quit command to the hint status server
+ cfg.setQuitStatusServer(true);
+ break;
+
default:
throw new MungeException("Unknown type of execution");
}
@@ -301,61 +421,87 @@ else if (cfg.isTargetsEnabled())
}
catch (Exception e)
{
+ fault = true;
if (logger != null)
{
- logger.error(e.getMessage());
+ logger.error(Utils.getStackTrace(e));
}
else
{
- System.out.println(e.getMessage());
+ System.out.println(Utils.getStackTrace(e));
}
isListening = false; // force stop
returnValue = 1;
}
finally
{
- if (!isListening)
+ // stop stuff
+ if (!isListening) // clients
{
+ // optionally command status server to quit
+ if (context.statusStty != null)
+ fault = context.statusStty.quitStatusServer(context, fault); // do before stopping the necessary services
+
+ // stop any remaining services
stopServices();
+
+ if (!cfg.getConsoleLevel().equalsIgnoreCase(cfg.getDebugLevel()))
+ logger.info("Log file has more details: " + cfg.getLogFilename());
+
Date done = new Date();
long millis = Math.abs(done.getTime() - stamp.getTime());
logger.fatal("Runtime: " + Utils.getDuration(millis));
+
+ if (!fault)
+ logger.fatal("Process completed normally");
+ else
+ logger.fatal("Process failed");
}
- else
+ else // daemons
{
+ // this shutdown hook is triggered when all connections and
+ // threads used by the daemon have been closed and stopped
+ // See ServeStty.run()
Runtime.getRuntime().addShutdownHook(new Thread()
{
public void run()
{
try
{
- if (context.clientStty != null)
- {
- context.clientStty.disconnect();
- }
+ // optionally command status server to quit
+ if (els.context.statusStty != null)
+ els.fault = els.context.statusStty.quitStatusServer(context, els.fault); // do before stopping the necessary services
+
+ if (!els.cfg.getConsoleLevel().equalsIgnoreCase(els.cfg.getDebugLevel()))
+ logger.info("Log file has more details: " + els.cfg.getLogFilename());
Date done = new Date();
long millis = Math.abs(done.getTime() - stamp.getTime());
logger.fatal("Runtime: " + Utils.getDuration(millis));
- logger.info("Stopping ELS services");
- Thread.sleep(10000L);
- stopServices();
+ if (!els.fault)
+ logger.fatal("Process completed normally");
+ else
+ logger.fatal("Process failed");
+
+ Thread.sleep(4000L);
+
+ // stop any remaining services
+ stopServices(); // has to be last
}
catch (Exception e)
{
- logger.error(e.getMessage() + "\r\n" + Utils.getStackTrace(e));
+ logger.error(Utils.getStackTrace(e));
}
}
});
}
}
-
return returnValue;
} // process
/**
- * Read either publisher or subscriber repository
+ * Read either a publisher or subscriber repository
*
* @param cfg Loaded configuration
* @param isPublisher Is this the publisher? true/false
@@ -449,6 +595,23 @@ else if (cfg.getSubscriberLibrariesFileName().length() == 0 && // n
*/
public void stopServices()
{
+ // logout from any hint status server if not shutting it down
+ if (context.statusStty != null)
+ {
+ if (!cfg.isQuitStatusServer() && context.statusStty.isConnected())
+ {
+ try
+ {
+ context.statusStty.send("logout");
+ }
+ catch (Exception e)
+ {
+ logger.error(Utils.getStackTrace(e));
+ }
+ }
+
+ context.statusStty.disconnect();
+ }
if (context.clientStty != null)
{
context.clientStty.disconnect();
diff --git a/src/com/groksoft/els/Process.java b/src/com/groksoft/els/Process.java
index 129e4da1..2026c26e 100755
--- a/src/com/groksoft/els/Process.java
+++ b/src/com/groksoft/els/Process.java
@@ -14,7 +14,7 @@
import java.util.ArrayList;
/**
- * ELS Process
+ * Process class where the primary operations are executed.
*/
public class Process
{
@@ -23,7 +23,6 @@ public class Process
private int differentSizes = 0;
private int errorCount = 0;
private boolean fault = false;
- private HintKeys hintKeys = null;
private Hints hints = null;
private ArrayList ignoredList = new ArrayList<>();
private boolean isInitialized = false;
@@ -175,7 +174,6 @@ private void munge() throws Exception
String header = "Munging collections " + context.publisherRepo.getLibraryData().libraries.description + " to " +
context.subscriberRepo.getLibraryData().libraries.description + (cfg.isDryRun() ? " (--dry-run)" : "");
- logger.info(header);
// setup the -m mismatch output file
if (cfg.getMismatchFilename().length() > 0)
@@ -213,6 +211,7 @@ private void munge() throws Exception
}
}
+ logger.info(header);
try
{
for (Library subLib : context.subscriberRepo.getLibraryData().libraries.bibliography)
@@ -413,7 +412,7 @@ private void munge() throws Exception
if (ignoredList.size() > 0)
{
- logger.info(SHORT, "+------------------------------------------");
+ logger.debug(SHORT, "+------------------------------------------");
logger.debug(SIMPLE, "Ignored " + ignoredList.size() + " files:");
for (String s : ignoredList)
{
@@ -471,12 +470,11 @@ private void munge() throws Exception
*
* What is done depends on the combination of options specified on the command line.
*/
- public int process()
+ public boolean process()
{
Marker SHORT = MarkerManager.getMarker("SHORT");
boolean lined = false;
boolean localHints = false;
- int returnValue = 0;
try
{
@@ -490,9 +488,9 @@ public int process()
// Get ELS hints keys if specified
if (cfg.getHintKeysFile().length() > 0) // v3.0.0
{
- hintKeys = new HintKeys(context);
- hintKeys.read(cfg.getHintKeysFile());
- hints = new Hints(cfg, context, hintKeys);
+ context.hintKeys = new HintKeys(cfg, context);
+ context.hintKeys.read(cfg.getHintKeysFile());
+ hints = new Hints(cfg, context, context.hintKeys);
}
// process ELS Hints locally, no subscriber, publisher's targets
@@ -540,7 +538,7 @@ public int process()
hints.hintsMunge();
}
- // if all the pieces are specified perform a full munge the collections
+ // if all the pieces are specified perform a full munge of the collections
if (!localHints && !cfg.isHintSkipMainProcess()) // v3.0.0
{
if (cfg.isTargetsEnabled() &&
@@ -571,7 +569,6 @@ public int process()
fault = true;
++errorCount;
logger.error(Utils.getStackTrace(ex));
- returnValue = 2;
}
finally
{
@@ -606,16 +603,10 @@ else if (resp == null)
logger.warn("Remote subscriber is in an unknown state");
}
}
-
- // mark the process as successful so it may be detected with automation
- if (!fault)
- logger.fatal(SHORT, "Process completed normally");
- else
- logger.fatal("Process failed");
}
}
- return returnValue;
+ return fault;
} // process
/**
@@ -631,6 +622,14 @@ private void rename() throws Exception
}
}
+ /**
+ * Dump any duplicates found to the log
+ *
+ * @param type Publisher or Subscriber, description
+ * @param item The item with duplicates
+ * @param duplicates The count of duplicated
+ * @return New count of duplicates
+ */
private int reportDuplicates(String type, Item item, int duplicates)
{
Marker SIMPLE = MarkerManager.getMarker("SIMPLE");
@@ -651,6 +650,14 @@ private int reportDuplicates(String type, Item item, int duplicates)
return duplicates;
}
+ /**
+ * Dump any empty directories to the log
+ *
+ * @param type Publisher or Subscriber, description
+ * @param item The item with empties
+ * @param empties The count of empties
+ * @return The new count of empties
+ */
private int reportEmpties(String type, Item item, int empties)
{
Marker SIMPLE = MarkerManager.getMarker("SIMPLE");
diff --git a/src/com/groksoft/els/Transfer.java b/src/com/groksoft/els/Transfer.java
index 7e00287b..23e813f5 100644
--- a/src/com/groksoft/els/Transfer.java
+++ b/src/com/groksoft/els/Transfer.java
@@ -19,7 +19,7 @@
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
/**
- * The Transfer class handles copying content to the appropriate location and
+ * Transfer class to handle copying content to the appropriate location and
* the local-only operations needed for ELS Hints.
*/
public class Transfer
@@ -42,6 +42,12 @@ public class Transfer
private Storage storageTargets = null;
private boolean toIsNew = false;
+ /**
+ * Constructor
+ *
+ * @param config Configuration
+ * @param ctx Context
+ */
public Transfer(Configuration config, Context ctx)
{
cfg = config;
@@ -137,11 +143,21 @@ public String copyGroup(ArrayList- group, long totalSize, boolean overwrite
return response;
}
+ /**
+ * Return the count of copies
+ *
+ * @return int count
+ */
public int getCopyCount()
{
return copyCount;
}
+ /**
+ * Return the current "group" name
+ *
+ * @return String group name
+ */
public String getCurrentGroupName()
{
return currentGroupName;
@@ -157,7 +173,7 @@ public String getCurrentGroupName()
public long getFreespace(String path) throws Exception
{
long space;
- if (cfg.isRemoteSession() && !context.hintMode)
+ if (cfg.isRemoteSession() && !context.hintMode)
{
// remote subscriber
space = context.clientStty.availableSpace(path);
@@ -169,21 +185,41 @@ public long getFreespace(String path) throws Exception
return space;
}
+ /**
+ * Get the grand total count of items
+ *
+ * @return int count
+ */
public long getGrandTotalItems()
{
return grandTotalItems;
}
+ /**
+ * Get the grand total of items copied to original locations
+ *
+ * @return int count
+ */
public long getGrandTotalOriginalLocation()
{
return grandTotalOriginalLocation;
}
+ /**
+ * Get the grand total of copied size
+ *
+ * @return long size in bytes
+ */
public long getGrandTotalSize()
{
return grandTotalSize;
}
+ /**
+ * Get the last group name
+ *
+ * @return String last group name
+ */
public String getLastGroupName()
{
return lastGroupName;
@@ -213,26 +249,51 @@ private long getLocationMinimum(String path)
return minimum;
}
+ /**
+ * Get the count of moved directories
+ *
+ * @return int count
+ */
public int getMovedDirectories()
{
return movedDirectories;
}
+ /**
+ * Get the count of moved files
+ *
+ * @return int count
+ */
public int getMovedFiles()
{
return movedFiles;
}
+ /**
+ * Get the count of removed directories
+ *
+ * @return int count
+ */
public int getRemovedDirectories()
{
return removedDirectories;
}
+ /**
+ * Get the count of removed files
+ *
+ * @return int count
+ */
public int getRemovedFiles()
{
return removedFiles;
}
+ /**
+ * Get the count of items skipped because they are missing
+ *
+ * @return int count
+ */
public int getSkippedMissing()
{
return skippedMissing;
@@ -258,7 +319,7 @@ private void getStorageTargets() throws Exception
cfg.setTargetsFilename(location);
}
- if (location != null) // v3.0.0 allow targets to be empty to use sources as target locations
+ if (location != null && location.length() > 0) // v3.0.0 allow targets to be empty to use sources as target locations
{
if (storageTargets == null)
storageTargets = new Storage();
@@ -603,31 +664,35 @@ private boolean moveItem(Repository repo, Library fromLib, Item fromItem, Librar
Item toItem = setupToItem(repo, fromLib, fromItem, toLib, toName);
// see if it still exists
- File fromFile = new File(fromItem.getFullPath());
+ String fromPath = repo.normalizePath(repo.getLibraryData().libraries.flavor, fromItem.getFullPath());
+ File fromFile = new File(fromPath);
if (fromFile.exists())
{
+ String toPath = repo.normalizePath(repo.getLibraryData().libraries.flavor, toItem.getFullPath());
+
if (cfg.isDryRun())
{
- logger.info(" > Would mv " + (fromItem.isDirectory() ? "directory " : "file ") + fromLib.name + "|" + fromItem.getItemPath() + " to " + toLib.name + "|" + toName);
+ logger.info(" > Would mv " + (fromItem.isDirectory() ? "directory " : "file ") +
+ "\"" + fromLib.name + "|" + fromPath + "\" to \"" + toLib.name + "|" + toPath + "\"");
return false;
}
// perform move / rename
- File toFile = new File(toItem.getFullPath());
+ File toFile = new File(toPath);
if (toFile.exists())
{
logger.info(" ! Target exists, will overwrite: " + toItem.getFullPath());
}
- else if (toIsNew)
+
+ // make sure the parent directories exist
+ if (toFile.getParentFile().mkdirs())
{
- if (toFile.getParentFile().mkdirs())
- {
- toLib.rescanNeeded = true;
- libAltered = true;
- }
+ toLib.rescanNeeded = true;
+ libAltered = true;
}
- logger.info(" > mv " + (fromItem.isDirectory() ? "directory " : "file ") + fromLib.name + "|" + fromItem.getItemPath() + " to " + toLib.name + "|" + toName);
+ logger.info(" > mv " + (fromItem.isDirectory() ? "directory " : "file ") +
+ "\"" + fromLib.name + "|" + fromPath + "\" to \"" + toLib.name + "|" + toPath + "\"");
Files.move(fromFile.toPath(), toFile.toPath(), REPLACE_EXISTING);
// no exception thrown
@@ -682,17 +747,18 @@ public boolean remove(Repository repo, String fromLibName, String fromName) thro
{
if (cfg.isDryRun())
{
- logger.info(" > Would rm directory: " + fromItem.getFullPath());
+ logger.info(" > Would rm directory: \"" + fromLibName + "|" + fromItem.getFullPath() + "\"");
}
else
{
// remove the physical directory
- File prevDir = new File(fromItem.getFullPath());
- if (Utils.removeDirectoryTree(prevDir))
+ String rmPath = repo.normalizePath(repo.getLibraryData().libraries.flavor, fromItem.getFullPath());
+ File rmdir = new File(rmPath);
+ if (Utils.removeDirectoryTree(rmdir))
{
logger.warn(" ! Previous directory was not empty: " + fromItem.getFullPath());
}
- logger.info(" > rm directory: " + fromItem.getFullPath());
+ logger.info(" > rm directory: \"" + fromItem.getFullPath() + "\"");
fromLib.rescanNeeded = true;
libAltered = true;
++removedDirectories;
@@ -702,14 +768,15 @@ public boolean remove(Repository repo, String fromLibName, String fromName) thro
{
if (cfg.isDryRun())
{
- logger.info(" > Would rm file: " + fromItem.getFullPath());
+ logger.info(" > Would rm file: " + fromLibName + "|" + fromItem.getFullPath());
}
else
{
- File prevFile = new File(fromItem.getFullPath());
- if (prevFile.delete())
+ String rmPath = repo.normalizePath(repo.getLibraryData().libraries.flavor, fromItem.getFullPath());
+ File rmFile = new File(rmPath);
+ if (rmFile.delete())
{
- logger.info(" > rm file: " + fromItem.getFullPath());
+ logger.info(" > rm file: \"" + fromItem.getFullPath() + "\"");
fromLib.rescanNeeded = true;
libAltered = true;
++removedFiles;
@@ -720,19 +787,26 @@ public boolean remove(Repository repo, String fromLibName, String fromName) thro
}
else
{
- logger.info(" ! Does not exist (A), skipping: " + fromLibName + "|" + fromName);
+ logger.info(" ! Does not exist (D), skipping: " + fromLibName + "|" + fromName);
++skippedMissing;
}
}
else
{
- logger.info(" ! Does not exist (B), skipping: " + fromLibName + "|" + fromName);
+ logger.info(" ! Does not exist (E), skipping: " + fromLibName + "|" + fromName);
++skippedMissing;
}
return libAltered;
}
+ /**
+ * Request the remote end re-scan and send it's collection JSON based on parameters
+ *
+ * Any -l | -L parameter is handled.
+ *
+ * @throws Exception
+ */
public void requestCollection() throws Exception
{
if (cfg.isRemoteSession())
diff --git a/src/com/groksoft/els/Utils.java b/src/com/groksoft/els/Utils.java
index 023df5c2..36193b92 100755
--- a/src/com/groksoft/els/Utils.java
+++ b/src/com/groksoft/els/Utils.java
@@ -9,6 +9,7 @@
import javax.crypto.spec.SecretKeySpec;
import java.io.*;
import java.math.BigDecimal;
+import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -22,7 +23,7 @@
import java.util.regex.Pattern;
/**
- * The type Utils. Various utility methods.
+ * Utils class of static utility methods.
*/
public class Utils
{
@@ -30,7 +31,7 @@ public class Utils
private static Logger logger = LogManager.getLogger("applog");
/**
- * Do not instantiate
+ * Static methods - do not instantiate
*/
private Utils()
{
@@ -126,6 +127,18 @@ public static byte[] encrypt(String key, String text)
return encrypted;
}
+ /**
+ * Format remote & local IP addresses and ports
+ *
+ * @param socket
+ * @return String of formatting information
+ */
+ public static String formatAddresses(Socket socket)
+ {
+ return socket.getInetAddress().toString() + ":" + socket.getPort() +
+ ", local " + socket.getLocalAddress().toString() + ":" + socket.getLocalPort();
+ }
+
/**
* Format a long number with byte, MB, GB and TB as applicable
*
@@ -257,6 +270,13 @@ public static String getLastPath(String full, String sep) throws MungeException
return path;
}
+ /**
+ * Get the path to the left of the filename
+ *
+ * @param full Full path to parse
+ * @param sep The directory separator for the local O/S
+ * @return String of left path
+ */
public static String getLeftPath(String full, String sep)
{
String path = "";
@@ -361,6 +381,21 @@ public static String getStackTrace(final Throwable throwable)
return sw.getBuffer().toString();
}
+ /**
+ * Is the path just a filename with no directory to the left?
+ *
+ * @param path Path to check
+ * @return true if it is just a filename
+ */
+ public static boolean isFileOnly(String path)
+ {
+ if (!path.contains("/") &&
+ !path.contains("\\") &&
+ !path.contains("|"))
+ return true;
+ return false;
+ }
+
/**
* Parse the host from a site string
*
@@ -466,7 +501,7 @@ public static String readStream(DataInputStream in, String key) throws Exception
{
if (e.getMessage().toLowerCase().contains("connection reset"))
{
- logger.info("connection closed by client");
+ logger.info("Connection closed by client");
input = null;
}
throw e;
@@ -503,6 +538,20 @@ public static boolean removeDirectoryTree(File directory)
return notAllDirectories;
}
+ /**
+ * Replace source pipe character with path separators
+ *
+ * @param repo Repository of source of path
+ * @param path Path to modify with pipe characters
+ * @return String Modified path
+ * @throws MungeException
+ */
+ public static String unpipe(Repository repo, String path) throws MungeException
+ {
+ String p = path.replaceAll("\\|", repo.getWriteSeparator());
+ return p;
+ }
+
/**
* Write an encrypted string to output stream
*
diff --git a/src/com/groksoft/els/repository/HintKeys.java b/src/com/groksoft/els/repository/HintKeys.java
index 6a31a181..9e620e53 100644
--- a/src/com/groksoft/els/repository/HintKeys.java
+++ b/src/com/groksoft/els/repository/HintKeys.java
@@ -1,19 +1,33 @@
package com.groksoft.els.repository;
+import com.groksoft.els.Configuration;
import com.groksoft.els.Context;
import com.groksoft.els.MungeException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
+/**
+ * Hint Keys class.
+ *
+ * Correlates the UUIDs in the publisher and subscriber JSON files
+ * with shorter names used inside hint .else files to track the
+ * status of completion for each defined node of an ELS system.
+ */
public class HintKeys
{
+ private Configuration cfg;
private Context context;
private String filename;
private ArrayList keys;
- public HintKeys(Context ctx)
+ private transient Logger logger = LogManager.getLogger("applog");
+
+ public HintKeys(Configuration config, Context ctx)
{
+ cfg = config;
context = ctx;
}
@@ -57,14 +71,17 @@ public void read(String file) throws Exception
throw new MungeException("Malformed line " + count + " reading ELS keys file: " + file);
}
- if (parts[1].equals(context.publisherRepo.getLibraryData().libraries.key))
+ if (!cfg.isStatusServer())
{
- foundPublisher = true;
- }
+ if (parts[1].equals(context.publisherRepo.getLibraryData().libraries.key))
+ {
+ foundPublisher = true;
+ }
- if (parts[1].equals(context.subscriberRepo.getLibraryData().libraries.key))
- {
- foundSubscriber = true;
+ if (parts[1].equals(context.subscriberRepo.getLibraryData().libraries.key))
+ {
+ foundSubscriber = true;
+ }
}
HintKey key = new HintKey();
@@ -77,16 +94,22 @@ public void read(String file) throws Exception
keys.add(key);
}
}
- if (!foundPublisher)
- throw new MungeException("The current publisher key was not found in ELS keys file: " + file);
- if (context.subscriberRepo != null && !foundSubscriber)
- throw new MungeException("The current subscriber key was not found in ELS keys file: " + file);
+
+ if (!cfg.isStatusServer())
+ {
+ if (!foundPublisher)
+ throw new MungeException("The current publisher key was not found in ELS keys file: " + file);
+ if (context.subscriberRepo != null && !foundSubscriber)
+ throw new MungeException("The current subscriber key was not found in ELS keys file: " + file);
+ }
+
+ logger.info("Read hints keys " + file + " successfully");
}
public class HintKey
{
- String name;
- String uuid;
+ public String name;
+ public String uuid;
}
}
diff --git a/src/com/groksoft/els/repository/Hints.java b/src/com/groksoft/els/repository/Hints.java
index 37ecbfe9..429c225b 100644
--- a/src/com/groksoft/els/repository/Hints.java
+++ b/src/com/groksoft/els/repository/Hints.java
@@ -1,6 +1,9 @@
package com.groksoft.els.repository;
-import com.groksoft.els.*;
+import com.groksoft.els.Configuration;
+import com.groksoft.els.Context;
+import com.groksoft.els.MungeException;
+import com.groksoft.els.Utils;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -16,7 +19,7 @@
import java.util.StringTokenizer;
/**
- * The Hints class handles finding and executing ELS Hints.
+ * Hints class to handle finding and executing ELS Hints and their commands.
*/
public class Hints
{
@@ -28,11 +31,20 @@ public class Hints
private int deletedHints = 0;
private int doneHints = 0;
private int executedHints = 0;
+ private String hintItemPath = "";
+ private String hintItemSubdirectory = "";
private HintKeys keys;
private int seenHints = 0;
private int skippedHints = 0;
private int validatedHints = 0;
+ /**
+ * Constructor
+ *
+ * @param config Configuration
+ * @param ctx Contect
+ * @param hintKeys HintKeys if enabled, else null
+ */
public Hints(Configuration config, Context ctx, HintKeys hintKeys)
{
cfg = config;
@@ -40,6 +52,9 @@ public Hints(Configuration config, Context ctx, HintKeys hintKeys)
keys = hintKeys;
}
+ /**
+ * Dump the statistics of an ELS Hints runs
+ */
private void dumpStats()
{
logger.info(SHORT, "+------------------------------------------");
@@ -49,11 +64,11 @@ private void dumpStats()
}
else
{
- logger.info(SHORT, "# Executed hints : " + executedHints);
- logger.info(SHORT, "# Done hints : " + doneHints); // TODO Reconsider metrics
- logger.info(SHORT, "# Seen hints : " + seenHints);
- logger.info(SHORT, "# Skipped hints : " + skippedHints);
+ logger.info(SHORT, "# Executed hints : " + executedHints + (cfg.isDryRun() ? " (--dry-run)" : ""));
+ logger.info(SHORT, "# Seen Hints : " + seenHints);
+ logger.info(SHORT, "# Done hints : " + doneHints);
logger.info(SHORT, "# Deleted hints : " + deletedHints);
+ logger.info(SHORT, "# Skipped hints : " + skippedHints);
logger.info(SHORT, "# Moved directories : " + context.transfer.getMovedDirectories());
logger.info(SHORT, "# Moved files : " + context.transfer.getMovedFiles());
logger.info(SHORT, "# Removed directories: " + context.transfer.getRemovedDirectories());
@@ -64,17 +79,15 @@ private void dumpStats()
logger.info(SHORT, "-------------------------------------------");
}
- private void dumpTerms(String[] parts)
- {
- for (int i = 0; i < parts.length; ++i)
- {
- if (parts[i] != null && parts[i].length() > 0)
- {
- logger.debug(" " + parts[i]);
- }
- }
- }
-
+ /**
+ * Execute the commands of an ELS Hint
+ *
+ * @param repo The Repository where the hint is executing
+ * @param item The hint Item
+ * @param lines The lines that have been read from the hints file
+ * @return true if the item's library was altered
+ * @throws Exception
+ */
private boolean execute(Repository repo, Item item, List lines) throws Exception
{
HintKeys.HintKey hintKey;
@@ -86,6 +99,8 @@ private boolean execute(Repository repo, Item item, List lines) throws E
// find the ELS key for this repo
hintKey = findHintKey(repo);
+ hintItemSubdirectory = item.getItemSubdirectory();
+
// find the actor name in the .els file
statusLine = findNameLine(lines, hintKey.name);
if (statusLine == null || !statusLine.toLowerCase().startsWith("for "))
@@ -112,7 +127,7 @@ private boolean execute(Repository repo, Item item, List lines) throws E
continue;
}
- if (line.startsWith("for ") || line.startsWith("done ") || line.startsWith("seen "))
+ if (line.startsWith("for ") || line.startsWith("done ") || line.startsWith("seen ") || line.startsWith("deleted "))
{
continue;
}
@@ -127,18 +142,24 @@ private boolean execute(Repository repo, Item item, List lines) throws E
if (fromLib == null)
fromLib = item.getLibrary(); // use the library of the .els item
- String fromName = parseFile(parts[1], lineNo);
+ String fromName = parseFilename(parts[1], lineNo);
if (fromName.length() < 1)
throw new MungeException("Malformed from filename on line " + lineNo);
+ if (hintItemSubdirectory != null && Utils.isFileOnly(fromName))
+ fromName = hintItemSubdirectory + "|" + fromName;
+
String toLib = parseLibrary(parts[2], lineNo);
if (toLib == null)
toLib = item.getLibrary(); // use the library of the .els item
- String toName = parseFile(parts[2], lineNo);
+ String toName = parseFilename(parts[2], lineNo);
if (toName.length() < 1)
throw new MungeException("Malformed to filename on line " + lineNo);
+ if (hintItemSubdirectory != null)
+ toName = hintItemSubdirectory + "|" + toName;
+
context.hintMode = true;
if (context.transfer.move(repo, fromLib.trim(), fromName.trim(), toLib.trim(), toName.trim()))
libAltered = true;
@@ -155,10 +176,13 @@ else if (line.toLowerCase().startsWith("rm ")) // rm remove
if (fromLib == null)
fromLib = item.getLibrary(); // use the library of the .els item
- String fromName = parseFile(parts[1], lineNo);
+ String fromName = parseFilename(parts[1], lineNo);
if (fromName.length() < 1)
throw new MungeException("Malformed from filename on line " + lineNo);
+ if (hintItemSubdirectory != null)
+ fromName = hintItemSubdirectory + "|" + fromName;
+
if (context.transfer.remove(repo, fromLib.trim(), fromName.trim()))
libAltered = true;
@@ -171,7 +195,7 @@ else if (line.toLowerCase().startsWith("rm ")) // rm remove
{
if (!cfg.isDryRun())
{
- updateHintStatus(item, lines, hintKey.name, "Done");
+ updateStatus(item, lines, hintKey.name, "Done");
}
++executedHints;
}
@@ -179,6 +203,13 @@ else if (line.toLowerCase().startsWith("rm ")) // rm remove
return libAltered;
}
+ /**
+ * Find the hint UUID key for a Repository
+ *
+ * @param repo Repository containing the UUID key to find
+ * @return Hints.Hintkey of matching UUID
+ * @throws Exception if not found
+ */
private HintKeys.HintKey findHintKey(Repository repo) throws Exception
{
// find the ELS key for this repo
@@ -190,6 +221,13 @@ private HintKeys.HintKey findHintKey(Repository repo) throws Exception
return hintKey;
}
+ /**
+ * Find the line in a hint .els file the matching the name
+ *
+ * @param lines Lines of the hint .els file
+ * @param name Name line to find
+ * @return String of matching name line or null if not found
+ */
private String findNameLine(List lines, String name)
{
for (String line : lines)
@@ -198,7 +236,7 @@ private String findNameLine(List lines, String name)
if (parts.length == 2)
{
String word = parts[0].toLowerCase();
- if (word.equals("for") || word.equals("done") || word.equals("seen"))
+ if (word.equals("for") || word.equals("done") || word.equals("seen") || word.equals("deleted"))
{
if (parts[1].equalsIgnoreCase(name))
return line.trim();
@@ -208,6 +246,13 @@ private String findNameLine(List lines, String name)
return null;
}
+ /**
+ * Get the target location to place a incoming hint file
+ *
+ * @param item Item to place
+ * @return String path of appropriate location
+ * @throws Exception
+ */
private String getHintTarget(Item item) throws Exception
{
String target = null;
@@ -255,7 +300,30 @@ private String getHintTarget(Item item) throws Exception
return target;
}
- public boolean hintExecute(String libName, String itemPath, String toPath) throws Exception
+ /**
+ * Get the String matching a given status rank
+ *
+ * @param rank Rank to match
+ * @return String of integer rank
+ */
+ private String getStatusString(int rank)
+ {
+ return ((rank == 0) ? "Unknown" : ((rank == 1) ? "For" : ((rank == 2) ? "Done" : (rank == 3 ? "Seen" : "Deleted"))));
+ }
+
+ /**
+ * Run a hint on a subscriber after merging status with an incoming .merge file.
+ *
+ * Used by the subscriber/Daemon when the command is sent by the publisher
+ * after the hint file has been copied to the subscriber.
+ *
+ * @param libName Library name of incoming item
+ * @param itemPath ItemPath of incoming item
+ * @param toPath Path of hint
+ * @return true if hint was executed
+ * @throws Exception
+ */
+ public boolean hintRun(String libName, String itemPath, String toPath) throws Exception
{
boolean sense = false;
Item toItem = null;
@@ -265,11 +333,13 @@ public boolean hintExecute(String libName, String itemPath, String toPath) throw
{
Item existingItem = context.subscriberRepo.hasItem(null, libName, itemPath);
if (existingItem != null)
+ {
toItem = SerializationUtils.clone(existingItem);
+ }
}
- // merge
- merge(toPath + ".merge", toPath);
+ // merge, might create the file
+ mergeHints(toPath + ".merge", toPath);
// execute
if (toItem == null)
@@ -283,17 +353,30 @@ public boolean hintExecute(String libName, String itemPath, String toPath) throw
Path entry = Paths.get(toPath);
toItem.setSize(Files.size(entry));
toItem.setFullPath(toPath);
+ if (!Utils.isFileOnly(toItem.getItemPath()))
+ {
+ toItem.setItemSubdirectory(Utils.pipe(context.subscriberRepo, Utils.getLeftPath(toItem.getItemPath(), context.subscriberRepo.getSeparator())));
+ }
- List lines = readHint(toItem);
+ List lines = readHintUpdated(toItem);
execute(context.subscriberRepo, toItem, lines);
if (toItem.isHintExecuted())
sense = true;
- postprocessHint(toItem);
+ postprocessHintFile(context.subscriberRepo, toItem);
return sense;
}
+ /**
+ * Run all the local hints on the publisher.
+ *
+ * If not done manually and the publisher's hint status set to Done, a hint
+ * must be executed on the publisher before a backup operation to a subscriber
+ * so the two ends match.
+ *
+ * @throws Exception
+ */
public void hintsLocal() throws Exception
{
boolean hintsFound = false;
@@ -324,9 +407,11 @@ public void hintsLocal() throws Exception
// check if it needs to be done locally
hintsFound = true;
- List lines = readHint(item);
+ List lines = readHintUpdated(item);
boolean libAltered = execute(context.publisherRepo, item, lines);
lib.rescanNeeded = true;
+
+ postprocessHintUpdated(context.subscriberRepo, item);
}
}
}
@@ -358,6 +443,15 @@ public void hintsLocal() throws Exception
}
}
+ /**
+ * Copy each publisher hint to a subscriber and have it execute the hint.
+ *
+ * Copies individual hints from the publisher to the subscriber, either
+ * locally or remotely, then commands the subscriber to execute the hint
+ * using hintrun().
+ *
+ * @throws Exception
+ */
public void hintsMunge() throws Exception
{
boolean hintsFound = false;
@@ -413,33 +507,43 @@ public void hintsMunge() throws Exception
// Hints are intended to be copied and processed immediately
// So the hint cannot be copied during a --dry-run
// Validate the syntax instead
- if (cfg.isDryRun()) // TODO How does --validate option fit in?
+ if (cfg.isDryRun())
{
logger.info("* Validating syntax for dry run: " + item.getFullPath());
++validatedHints;
- List lines = readHint(item); // reads & validates hint
+ List lines = readHintUpdated(item); // reads & validates hint
}
else
{
// read the ELS hint file
- List lines = readHint(item);
+ List lines = readHint(item.getFullPath());
// check if the publisher has Done or Seen this hint
// this is important prior to a backup run to avoid duplicates, etc.
HintKeys.HintKey hintKey = findHintKey(context.publisherRepo);
String statusLine = findNameLine(lines, hintKey.name);
if (statusLine == null || (!statusLine.toLowerCase().startsWith("done ") && !statusLine.toLowerCase().startsWith("seen ")))
- throw new MungeException("Publisher must execute hints locally; Status is not Done or Seen in hint: " + item.getFullPath());
+ throw new MungeException("Publisher must execute hints locally; Status has not Done or Seen hint: " + item.getFullPath());
+
+ if (statusLine.toLowerCase().startsWith("done "))
+ ++doneHints;
+ else if (statusLine.toLowerCase().startsWith("seen "))
+ ++seenHints;
String toPath = getHintTarget(item); // never null
String tmpPath = toPath + ".merge";
context.transfer.copyFile(item.getFullPath(), tmpPath, true);
+ toItem = SerializationUtils.clone(item);
+ toItem.setFullPath(toPath);
+ if (!Utils.isFileOnly(toItem.getItemPath()))
+ {
+ toItem.setItemSubdirectory(Utils.pipe(context.subscriberRepo, Utils.getLeftPath(toItem.getItemPath(), context.subscriberRepo.getSeparator())));
+ }
+
boolean updatePubSide = false;
if (cfg.isRemoteSession())
{
- toItem = SerializationUtils.clone(item);
- toItem.setFullPath(toPath);
logger.info("* Executing " + item.getFullPath() + " remotely on " + context.subscriberRepo.getLibraryData().libraries.description);
// Send command to merge & execute
@@ -447,12 +551,12 @@ public void hintsMunge() throws Exception
toItem.getLibrary() + "\" \"" + toItem.getItemPath() + "\" \"" + toPath + "\"");
if (response != null && response.length() > 0)
{
- logger.debug(" > execute command returned: " + response);
+ logger.info(" > execute command returned: " + response);
updatePubSide = response.equalsIgnoreCase("true") ? true : false;
if (updatePubSide)
{
subLib.rescanNeeded = true;
- ++executedHints;
+ //++executedHints;
}
}
else
@@ -461,25 +565,26 @@ public void hintsMunge() throws Exception
else
{
// merge
- merge(tmpPath, toPath);
+ mergeHints(tmpPath, toPath);
// execute
- toItem = SerializationUtils.clone(item);
- toItem.setFullPath(toPath);
-
- lines = readHint(toItem);
+ lines = readHintUpdated(toItem);
execute(context.subscriberRepo, toItem, lines);
if (toItem.isHintExecuted())
+ {
+ subLib.rescanNeeded = true;
+ //++executedHints;
updatePubSide = true;
+ }
}
- updateHintSubscriberOnPublisher(item);
+ updateSubscriberOnPublisher(item);
if (!cfg.isRemoteSession()) // subscriber-side does this itself
{
- postprocessHint(toItem);
+ postprocessHintFile(context.subscriberRepo, toItem);
}
- postprocessHint(item);
+ postprocessHintUpdated(context.publisherRepo, item);
}
}
}
@@ -536,13 +641,22 @@ public void hintsMunge() throws Exception
}
}
+ /**
+ * Clean-up hint status and files on subscriber.
+ *
+ * Checks each hint file on a subscriber to promote the status
+ * from Done to Seen and potentially deletes the hint file if
+ * the status allows it - for automatic hint maintenance.
+ *
+ * @throws Exception
+ */
public void hintsSubscriberCleanup() throws Exception
{
if (cfg.isRemoteSession() && !context.hintMode)
{
- logger.info("Sending hints cleanup command to remote " + context.subscriberRepo.getLibraryData().libraries.description);
+ logger.debug("Sending hints cleanup command to remote: " + context.subscriberRepo.getLibraryData().libraries.description);
- // Send command to merge & execute
+ // Send command to clean-up hints
String response = context.clientStty.roundTrip("cleanup");
if (response != null && response.length() > 0)
{
@@ -559,7 +673,19 @@ public void hintsSubscriberCleanup() throws Exception
}
}
- private void merge(String mergePath, String toPath) throws Exception
+ /**
+ * Merge an incoming hint file with any existing hint file.
+ *
+ * Merges the completion status of each back-up in an existing
+ * hint file with that of a hint file coming from the publisher.
+ * The highest completion status is used. If no hint file exist
+ * a new file is created.
+ *
+ * @param mergePath The path of the incoming .els.merge file
+ * @param toPath The resulting path of the merged hint
+ * @throws Exception
+ */
+ private void mergeHints(String mergePath, String toPath) throws Exception
{
File toFile = new File(toPath);
if (!toFile.exists())
@@ -580,18 +706,18 @@ private void merge(String mergePath, String toPath) throws Exception
{
String mergeline = mergeLines.get(i);
String copy = mergeline.toLowerCase();
- if (copy.startsWith("for ") || copy.startsWith("done ") || copy.startsWith("seen "))
+ if (copy.startsWith("for ") || copy.startsWith("done ") || copy.startsWith("seen ") || copy.startsWith("deleted "))
{
String[] mergeParts = parseNameLine(mergeline, i);
String mergeStatus = mergeParts[0];
String mergeName = mergeParts[1];
- int mergeRank = statusToInt(mergeStatus);
+ int mergeRank = statusToRank(mergeStatus);
for (int j = 0; j < toLines.size(); ++j)
{
String existing = toLines.get(j);
copy = existing.toLowerCase();
- if (copy.startsWith("for ") || copy.startsWith("done ") || copy.startsWith("seen "))
+ if (copy.startsWith("for ") || copy.startsWith("done ") || copy.startsWith("seen ") || copy.startsWith("deleted "))
{
String[] existingParts = parseNameLine(existing, j);
String existingStatus = existingParts[0];
@@ -599,14 +725,15 @@ private void merge(String mergePath, String toPath) throws Exception
if (mergeName.equalsIgnoreCase(existingName))
{
- int existingRank = statusToInt(existingStatus);
+ int existingRank = statusToRank(existingStatus);
if (mergeRank > existingRank)
{
- mergeStatus = (mergeRank == 0) ? "For" : ((mergeRank == 1) ? "Done" : "Seen");
+ mergeStatus = getStatusString(mergeRank);
mergeline = mergeStatus + " " + mergeName;
mergeLines.set(i, mergeline);
changed = true;
}
+ break;
}
}
}
@@ -622,6 +749,84 @@ private void merge(String mergePath, String toPath) throws Exception
mergeFile.delete();
}
+ /**
+ * Merge values from the ELS Hint Tracker with those in a hint file.
+ *
+ * If hint tracking is being used, where a --hint-server is defined
+ * for either a local or remote operation, then it's status values are
+ * merged with the hint file.
+ *
+ * @param item Item of the hint file
+ * @param lines Lines of the hint file
+ * @return Merged lines of the hint file
+ * @throws Exception
+ */
+ private List mergeStatusServer(Item item, List lines) throws Exception
+ {
+ // is a hint status server being used?
+ if (cfg.isUsingHintTracker())
+ {
+ boolean changed = false;
+ for (int i = 0; i < lines.size(); ++i)
+ {
+ String existing = lines.get(i);
+ String copy = existing.toLowerCase();
+ if (copy.startsWith("for ") || copy.startsWith("done ") || copy.startsWith("seen ") || copy.startsWith("deleted "))
+ {
+ String[] existingParts = parseNameLine(existing, i);
+ String existingStatus = existingParts[0];
+ String existingName = existingParts[1];
+ int existingRank = statusToRank(existingStatus);
+ int mergeRank;
+
+ if (cfg.isRemoteSession())
+ //if (context.statusStty != null && context.statusStty.isConnected())
+ {
+ // get the status from the status server
+ String command = "get \"" + item.getLibrary() + "\" " +
+ "\"" + item.getItemPath() + "\" " +
+ "\"" + existingName + "\" " +
+ "\"" + existingStatus + "\"";
+
+ String response = context.statusStty.roundTrip(command);
+ if (response != null && !response.equalsIgnoreCase("false"))
+ {
+ mergeRank = statusToRank(response);
+ }
+ else
+ throw new MungeException("Status Server " + context.statusRepo.getLibraryData().libraries.description +
+ " failure during get, line: " + i + " in: " + item.getFullPath());
+ }
+ else
+ {
+ String mergeStatus = context.datastore.getStatus(item.getLibrary(), item.getItemPath(), existingName, existingStatus);
+ mergeRank = statusToRank(mergeStatus);
+ }
+
+ if (mergeRank > existingRank)
+ {
+ String mergeStatus = getStatusString(mergeRank);
+ lines.set(i, mergeStatus + " " + existingName);
+ changed = true;
+ }
+ }
+ }
+
+ if (changed)
+ Files.write(Paths.get(item.getFullPath()), lines, StandardOpenOption.CREATE);
+ }
+ return lines;
+ }
+
+ /**
+ * Parse a hint file command line
+ *
+ * @param line Line to parse
+ * @param lineNo The number of the line in the file
+ * @param expected The number of expected values
+ * @return String[] of arguments
+ * @throws Exception
+ */
private String[] parseCommand(String line, int lineNo, int expected) throws Exception
{
int MAX_TERMS = 4;
@@ -646,12 +851,20 @@ private String[] parseCommand(String line, int lineNo, int expected) throws Exce
return cmd;
}
- private String parseFile(String term, int lineNo) throws MungeException
+ /**
+ * Parse the right-side filename portion of a hint command line
+ *
+ * @param line The line to parse
+ * @param lineNo The number of the line in the file
+ * @return String the parsed filename or null
+ * @throws MungeException
+ */
+ private String parseFilename(String line, int lineNo) throws MungeException
{
String name = null;
- String[] parts = term.split("\\|");
+ String[] parts = line.split("\\|");
if (parts.length > 2)
- throw new MungeException("Malformed library|file term on line " + lineNo + ": " + term);
+ throw new MungeException("Malformed library|file term on line " + lineNo + ": " + line);
if (parts.length == 1)
name = parts[0];
else if (parts.length == 2)
@@ -659,17 +872,33 @@ else if (parts.length == 2)
return name;
}
- private String parseLibrary(String term, int lineNo) throws MungeException
+ /**
+ * Parse the left-side library name portion of a hint command line
+ *
+ * @param line The line to parse
+ * @param lineNo The number of the line in the file
+ * @return String the parsed library or null
+ * @throws MungeException
+ */
+ private String parseLibrary(String line, int lineNo) throws MungeException
{
String lib = null;
- String[] parts = term.split("\\|");
+ String[] parts = line.split("\\|");
if (parts.length > 2)
- throw new MungeException("Malformed library|file term on line " + lineNo + ": " + term);
+ throw new MungeException("Malformed library|file term on line " + lineNo + ": " + line);
if (parts.length == 2)
lib = parts[0];
return lib;
}
+ /**
+ * Parse a hint file name/status line
+ *
+ * @param line The line to parse
+ * @param lineNo The number of the line in the file
+ * @return String[2] of back-up name and status
+ * @throws Exception
+ */
private String[] parseNameLine(String line, int lineNo) throws Exception
{
String[] cmd = new String[2];
@@ -691,24 +920,47 @@ private String[] parseNameLine(String line, int lineNo) throws Exception
return cmd;
}
- private void postprocessHint(Item item) throws Exception
+ /**
+ * Post-process a hint's status.
+ *
+ * Scans the lines of a hint promoting status as the hint goes through
+ * the steps of For, Done, Seen then Deleted. When all back-up's status
+ * is either Seen or Deleted the hint file is deleted for automatic
+ * hint maintenance.
+ *
+ * @param repo The Repository containing the hint file
+ * @param item The Item of the hint file
+ * @param lines The lines of the hint file
+ * @return String the current/updated status of this hint
+ * @throws Exception
+ */
+ private String postprocessHint(Repository repo, Item item, List lines) throws Exception
{
int doneWords = 0;
int forWords = 0;
- int seenWords = 0;
+ int deletedWords = 0;
String pubStat = "";
+ int seenWords = 0;
String subStat = "";
int totalWords = 0;
-
- // read the ELS hint file
- if (Files.notExists(Paths.get(item.getFullPath())))
- return;
- List lines = readHint(item.getFullPath()); // hint not validated
+ String currentStat = "";
+ boolean isPub = false;
// find the ELS keys
HintKeys.HintKey pubHintKey = findHintKey(context.publisherRepo);
HintKeys.HintKey subHintKey = findHintKey(context.subscriberRepo);
+ HintKeys.HintKey itemHintKey;
+ if (repo == context.publisherRepo)
+ {
+ itemHintKey = pubHintKey;
+ isPub = true;
+ }
+ else if (repo == context.subscriberRepo)
+ itemHintKey = subHintKey;
+ else
+ throw new MungeException("Unknown repo: " + repo.getLibraryData().libraries.description);
+ // scan the lines adding-up status values
for (String line : lines)
{
String parts[] = line.split("[\\s]+");
@@ -719,12 +971,16 @@ private void postprocessHint(Item item) throws Exception
if (name.equalsIgnoreCase(pubHintKey.name))
{
pubStat = word;
+ if (isPub)
+ currentStat = word;
}
if (name.equalsIgnoreCase(subHintKey.name))
{
subStat = word;
+ if (!isPub)
+ currentStat = word;
}
- if (word.equals("for") || word.equals("done") || word.equals("seen"))
+ if (word.equals("for") || word.equals("done") || word.equals("seen") || word.equals("deleted"))
{
++totalWords;
switch (word)
@@ -738,48 +994,109 @@ private void postprocessHint(Item item) throws Exception
case "seen":
++seenWords;
break;
+ case "deleted":
+ ++deletedWords;
+ break;
}
}
}
}
- if (forWords == 0)
+ if (forWords == 0) // if it is still "For" any back-up don't do anything
{
- if (doneWords > 0)
+ if (doneWords > 0) // if some for "Done" ...
{
- if (pubStat.equals("done"))
+ if (pubStat.equals("done")) // if the publisher is "Done" promote to "Seen"
{
- updateHintStatus(item, lines, pubHintKey.name, "Seen");
+ updateStatus(item, lines, pubHintKey.name, "Seen");
+ if (isPub)
+ currentStat = "Seen";
--doneWords;
++seenWords;
}
- if (subStat.equals("done"))
+
+ // if the subscriber is "Done" promote to "Seen"
+ // skip if this is hintsLocal() where they are the same Repository
+ if (context.publisherRepo != context.subscriberRepo)
{
- updateHintStatus(item, lines, subHintKey.name, "Seen");
- --doneWords;
- ++seenWords;
+ if (subStat.equals("done"))
+ {
+ updateStatus(item, lines, subHintKey.name, "Seen");
+ if (!isPub)
+ currentStat = "Seen";
+ --doneWords;
+ ++seenWords;
+ }
}
}
- else if (seenWords == totalWords)
+
+ // if all the back-ups have either "Seen" or "Deleted" the hint then
+ // all back-ups have "Done" it so delete the hint file
+ if (seenWords + deletedWords == totalWords)
{
File prevFile = new File(item.getFullPath());
if (prevFile.exists())
{
if (cfg.isDryRun())
{
- logger.info(" > hint done and seen, would remove: " + item.getFullPath());
+ logger.info(" > Hint done and seen, would delete hint file: " + item.getFullPath());
}
else
{
if (prevFile.delete())
{
- logger.info(" > hint done and seen, removing: " + item.getFullPath());
+ logger.info(" > Hint done and seen, deleted hint file: " + item.getFullPath());
++deletedHints;
+ currentStat = "Deleted";
+ updateStatusServer(item, itemHintKey.name, "Deleted");
+ repo.getLibrary(item.getLibrary()).rescanNeeded = true;
}
}
}
}
}
+ return currentStat;
+ }
+
+ /**
+ * Post-process a hint file.
+ *
+ * If a hint file still exists use postprocessHint() to update
+ * any appropriate status values.
+ *
+ * @param repo The Repository of the hint
+ * @param item The Item of the hint
+ * @return String the current/updated status of this hint
+ * @throws Exception
+ */
+ private String postprocessHintFile(Repository repo, Item item) throws Exception
+ {
+ // read the ELS hint file
+ if (Files.notExists(Paths.get(item.getFullPath())))
+ return "Deleted";
+ List lines = readHint(item.getFullPath()); // hint not validated
+ return postprocessHint(repo, item, lines);
+ }
+
+ /**
+ * Post-process a hint file updated from the Hint Tracker/Server.
+ *
+ * If a hint file still exists updated it with values from the Hint Tracker
+ * or Hint Server, if defined, then use postprocessHint() to update
+ * any appropriate status values.
+ *
+ * @param repo The Repository of the hint
+ * @param item The Item of the hint
+ * @return String the current/updated status of this hint
+ * @throws Exception
+ */
+ private String postprocessHintUpdated(Repository repo, Item item) throws Exception
+ {
+ // read the ELS hint file
+ if (Files.notExists(Paths.get(item.getFullPath())))
+ return "Deleted";
+ List lines = readHintUpdated(item);
+ return postprocessHint(repo, item, lines);
}
/**
@@ -787,7 +1104,6 @@ else if (seenWords == totalWords)
*
* @param path Full path to hint file
* @return String lines that have tabs replaced with a space and trimmed
- * @throws Exception
*/
private List readHint(String path) throws Exception
{
@@ -804,27 +1120,48 @@ private List readHint(String path) throws Exception
}
/**
- * Read, cleanup & validate lines from a hint file
+ * Read, cleanup, merge with status tracker/server & validate lines from a hint file
*
* @param item Item to be read
- * @return String lines that have tabs replaced with a space, trimmed and are validated
+ * @return String lines
* @throws Exception
*/
- private List readHint(Item item) throws Exception
+ private List readHintUpdated(Item item) throws Exception
{
List lines = readHint(item.getFullPath());
+ lines = mergeStatusServer(item, lines);
return validate(item.getFullPath(), lines);
}
- private int statusToInt(String status)
+ /**
+ * Return the numeric rank of a status String value
+ *
+ * @param status The value to rank
+ * @return int The numeric rank of the status String, or 0 if no match
+ */
+ private int statusToRank(String status)
{
if (status.equalsIgnoreCase("for"))
- return 0;
- else if (status.equalsIgnoreCase("done"))
return 1;
- return 2;
+ else if (status.equalsIgnoreCase("done"))
+ return 2;
+ else if (status.equalsIgnoreCase("seen"))
+ return 3;
+ else if (status.equalsIgnoreCase("deleted"))
+ return 4;
+ return 0;
}
+ /**
+ * Clean-up a local subscriber's hint files.
+ *
+ * Used at the end of an operation either locally or by the
+ * subscriber/Daemon when the command is received from the
+ * publisher. Scans the subscriber for .els files then runs
+ * postprocessHintFile() on each.
+ *
+ * @throws Exception
+ */
private void subscriberCleanup() throws Exception
{
logger.info("Cleaning-up ELS Hints on " + context.subscriberRepo.getLibraryData().libraries.description);
@@ -859,7 +1196,12 @@ private void subscriberCleanup() throws Exception
{
continue;
}
- postprocessHint(item);
+ File itemFile = new File(item.getFullPath());
+ if (itemFile.exists())
+ {
+ List lines = readHintUpdated(item); // merge with status server if in use
+ postprocessHintFile(context.subscriberRepo, item);
+ }
}
}
}
@@ -870,44 +1212,140 @@ private void subscriberCleanup() throws Exception
}
}
- private void updateHintStatus(Item item, List lines, String name, String status) throws Exception
+ /**
+ * Update hint status for a specific back-up name.
+ *
+ * Updates and saves the status for the backup name in the hint
+ * file and updates the hint status tracker/server if defined.
+ *
+ * The higher value of either the existing or the new value is saved.
+ *
+ * @param item The Item of the hint
+ * @param lines The lines of the hint
+ * @param backupName The name of the back-up from the Hint Keys file
+ * @param status The new status for the hint
+ * @return String of the value actually saved
+ * @throws Exception
+ */
+ private String updateStatus(Item item, List lines, String backupName, String status) throws Exception
{
boolean changed = false;
int lineNo = 0;
+
+ int mergeRank = statusToRank(status);
for (String line : lines)
{
String[] parts = line.split("[\\s]+");
if (parts.length == 2)
{
String word = parts[0].toLowerCase();
- if (word.equals("for") || word.equals("done") || word.equals("seen"))
+ if (word.equals("for") || word.equals("done") || word.equals("seen") || word.equals("deleted"))
{
- int rank = statusToInt(word);
- int toRank = statusToInt(status);
- if (toRank > rank)
+ if (parts[1].equalsIgnoreCase(backupName))
{
- if (parts[1].equalsIgnoreCase(name))
+ int existingRank = statusToRank(word);
+ if (mergeRank > existingRank)
{
- line = status + " " + parts[1];
+ line = status + " " + backupName;
lines.set(lineNo, line);
changed = true;
+ break;
+ }
+ else
+ {
+ status = word;
+ break;
}
}
}
}
++lineNo;
}
+
if (changed)
+ {
Files.write(Paths.get(item.getFullPath()), lines, StandardOpenOption.CREATE);
+ updateStatusServer(item, backupName, status);
+ }
+ return status;
}
- private void updateHintSubscriberOnPublisher(Item item) throws Exception
+ /**
+ * Update the hint status tracker/server.
+ *
+ * If defined on the command line with -H | --hint-server the tracker
+ * is updated either locally or remotely when the -r | --remote option
+ * is used.
+ *
+ * No further processing of the status is done by the tracker/server,
+ * i.e. the status is not changed.
+ *
+ * @param item The Item of the hint
+ * @param backupName The name of the back-up from the ELS Hint Keys file
+ * @param status The desired status String
+ * @throws Exception
+ */
+ private void updateStatusServer(Item item, String backupName, String status) throws Exception
{
- List lines = readHint(item.getFullPath()); // hint not validated
+ // is a hint status server being used?
+ if (cfg.isUsingHintTracker())
+ {
+ if (cfg.isRemoteSession())
+ {
+ // set the status on the status server
+ String command = "set \"" + item.getLibrary() + "\" " +
+ "\"" + item.getItemPath() + "\" " +
+ "\"" + backupName + "\" " +
+ "\"" + status + "\"";
+
+ String response = context.statusStty.roundTrip(command);
+ if (response == null || !response.equalsIgnoreCase(status))
+ throw new MungeException("Status Server " + context.statusRepo.getLibraryData().libraries.description + " returned a failure during set");
+ }
+ else
+ {
+ String result = context.datastore.setStatus(item.getLibrary(), item.getItemPath(), backupName, status);
+ if (result == null || !result.equalsIgnoreCase(status))
+ throw new MungeException("Hint setStatus() for " + context.statusRepo.getLibraryData().libraries.description + " returned a failure during set");
+ }
+ }
+ }
+
+ /**
+ * Update the subscriber's status in the publisher's hint file.
+ *
+ * Merges any existing status with "Done". The highest value is saved.
+ *
+ * @param item The Item of the hint
+ * @return Resulting String status
+ * @throws Exception
+ */
+ private String updateSubscriberOnPublisher(Item item) throws Exception
+ {
+ String currentStat = "";
+ List lines = readHintUpdated(item);
HintKeys.HintKey hintKey = findHintKey(context.subscriberRepo);
- updateHintStatus(item, lines, hintKey.name, "Done");
+ String line = findNameLine(lines, hintKey.name);
+ if (line != null)
+ {
+ String[] parts = line.split("[\\s]+"); // two parts guaranteed
+ int rank = statusToRank(parts[0]);
+ int toRank = statusToRank("Done");
+ if (rank > toRank)
+ toRank = rank;
+ return updateStatus(item, lines, hintKey.name, getStatusString(toRank));
+ }
+ return currentStat;
}
+ /**
+ * Validate the syntax of a hint file
+ *
+ * @param filename The file path of the hint file
+ * @param lines The lines of the hint file
+ * @return The validated lines of the hint file
+ * @throws Exception Any problem found throws an exception
+ */
private List validate(String filename, List lines) throws Exception
{
int lineNo = 0;
@@ -928,14 +1366,14 @@ private List validate(String filename, List lines) throws Except
lowered = line.toLowerCase();
- if (lowered.startsWith("for ") || lowered.startsWith("done ") || lowered.startsWith("seen "))
+ if (lowered.startsWith("for ") || lowered.startsWith("done ") || lowered.startsWith("seen ") || lowered.startsWith("deleted "))
{
- if (lowered.startsWith("done "))
- ++doneHints;
- else if (lowered.startsWith("seen "))
- ++seenHints;
-
- continue;
+ String[] parts = line.split("[\\s]+");
+ if (parts != null && parts.length == 2)
+ {
+ if (parts[0].length() > 0 && parts[1].length() > 0)
+ continue;
+ }
}
if (lowered.startsWith("mv "))
@@ -945,7 +1383,7 @@ else if (lowered.startsWith("seen "))
String fromName = "";
if (parts != null && parts.length > 2)
{
- fromName = parseFile(parts[1], lineNo);
+ fromName = parseFilename(parts[1], lineNo);
String fromLib = parseLibrary(parts[1], lineNo);
}
if (!(fromName.length() > 0))
@@ -956,7 +1394,7 @@ else if (lowered.startsWith("seen "))
if (parts.length > 3)
{
toLib = parseLibrary(parts[2], lineNo);
- toName = parseFile(parts[2], lineNo);
+ toName = parseFilename(parts[2], lineNo);
}
if (!(toName.length() > 0))
throw new MungeException("Malformed to filename on line " + lineNo + " in " + filename);
@@ -969,8 +1407,8 @@ else if (lowered.startsWith("seen "))
String[] parts = parseCommand(line, lineNo, 2);
if (parts != null && parts.length > 2)
- parseLibrary(parts[1], lineNo);
- String fromName = parseFile(parts[1], lineNo);
+ parseLibrary(parts[1], lineNo);
+ String fromName = parseFilename(parts[1], lineNo);
if (!(fromName.length() > 0))
throw new MungeException("Malformed from filename on line " + lineNo + " in " + filename);
diff --git a/src/com/groksoft/els/repository/Item.java b/src/com/groksoft/els/repository/Item.java
index e5c00d95..53246971 100644
--- a/src/com/groksoft/els/repository/Item.java
+++ b/src/com/groksoft/els/repository/Item.java
@@ -14,10 +14,12 @@ public class Item implements Serializable
private transient List- hasList = null;
private transient boolean hintExecuted = false;
private String itemPath;
+ private transient String itemSubdirectory;
private String library;
private transient boolean reported = false;
private long size = -1L;
private boolean symLink = false;
+
/**
* Instantiates a new Item.
*/
@@ -30,7 +32,7 @@ public Item()
/**
* Add has.
*
- * @param a matching item
+ * @param item The item to add
*/
public void addHas(Item item)
{
@@ -91,6 +93,26 @@ public void setItemPath(String itemPath)
this.itemPath = itemPath;
}
+ /**
+ * Get the item's subdirectory within the library.
+ *
+ * @return String of subdirectory or null
+ */
+ public String getItemSubdirectory()
+ {
+ return itemSubdirectory;
+ }
+
+ /**
+ * Set the item's subdirectory within the library.
+ *
+ * @param itemSubdirectory
+ */
+ public void setItemSubdirectory(String itemSubdirectory)
+ {
+ this.itemSubdirectory = itemSubdirectory;
+ }
+
/**
* Gets library.
*
@@ -155,11 +177,21 @@ public void setDirectory(boolean directory)
this.directory = directory;
}
+ /**
+ * Has this hint Item been executed?
+ *
+ * @return true if executed
+ */
public boolean isHintExecuted()
{
return hintExecuted;
}
+ /**
+ * Set the value of this hint Item being executed
+ *
+ * @param hintExecuted
+ */
public void setHintExecuted(boolean hintExecuted)
{
this.hintExecuted = hintExecuted;
diff --git a/src/com/groksoft/els/repository/Libraries.java b/src/com/groksoft/els/repository/Libraries.java
index d2f766c7..4f978cd1 100644
--- a/src/com/groksoft/els/repository/Libraries.java
+++ b/src/com/groksoft/els/repository/Libraries.java
@@ -9,70 +9,58 @@
*/
public class Libraries
{
- public static final String WINDOWS = "windows";
- public static final String LINUX = "linux";
public static final String APPLE = "apple";
-
+ public static final String LINUX = "linux";
+ public static final String WINDOWS = "windows";
+ /**
+ * The list of libraries.
+ */
+ public Library[] bibliography;
+ /**
+ * If case-sensitive true/false.
+ */
+ public Boolean case_sensitive;
/**
* Compiled patterns of ignore_patterns.
*/
public transient List compiledPatterns = new ArrayList<>();
-
/**
* The Description of this set of libraries.
*/
public String description;
-
/**
- * The host for outgoing connections, [host name|IP address]:[port]
- * Default port is 50271 if not specified
+ * Flavor of system: Windows, Linux, or Mac (only)
*/
- public String host;
-
+ public String flavor;
/**
- * The listen for incoming connections, [host name|IP address]:[port]
+ * The host for outgoing connections, [host name|IP address]:[port]
* Default port is 50271 if not specified
*/
- public String listen;
-
- /**
- * Flavor of system: Windows, Linux, or Mac (only)
- */
- public String flavor;
-
+ public String host;
/**
- * If remote terminal session is allowed then true, else false
+ * Ignore patterns. Regular expressions are supported.
*/
- public String terminal_allowed;
-
+ public String[] ignore_patterns;
/**
* The UUID of this system
*/
public String key;
-
/**
- * If case-sensitive true/false.
+ * The listen for incoming connections, [host name|IP address]:[port]
+ * Default port is 50271 if not specified
*/
- public Boolean case_sensitive;
-
+ public String listen;
/**
- * Ignore patterns. Regular expressions are supported.
+ * Storage. v3.0.0
*/
- public String[] ignore_patterns;
-
+ public Location[] locations;
/**
* Substitutions. From-side regular expressions are supported.
*/
public Renaming[] renaming;
-
- /**
- * Storage. v3.0.0
- */
- public Location[] locations;
-
/**
- * The list of libraries.
+ * If remote terminal session is allowed then true, else false
*/
- public Library[] bibliography;
+ public Boolean terminal_allowed;
}
diff --git a/src/com/groksoft/els/repository/Library.java b/src/com/groksoft/els/repository/Library.java
index cc87b064..a7c600f1 100644
--- a/src/com/groksoft/els/repository/Library.java
+++ b/src/com/groksoft/els/repository/Library.java
@@ -1,7 +1,7 @@
package com.groksoft.els.repository;
import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
+
import java.util.Vector;
/**
@@ -11,28 +11,41 @@ public class Library
{
/**
* Transient hash map for item look-ups
+ *
* @see ArrayListMultimap class API doc
*/
public transient ArrayListMultimap itemMap;
-
/**
- * Library has been altered, transient
+ * One or more Items. Last member so name appears first in data.
*/
- public transient boolean rescanNeeded = false;
-
+ public Vector
- items;
/**
* The library Name.
*/
public String name;
-
+ /**
+ * Library has been altered, transient
+ */
+ public transient boolean rescanNeeded = false;
/**
* One or more Sources.
*/
public String[] sources;
/**
- * One or more Items. Last member so name appears first in data.
+ * Get an item by itemPath in a linear search
+ *
+ * @param itemPath
+ * @return Item matching itemPath
*/
- public Vector
- items;
+ public Item get(String itemPath)
+ {
+ for (Item item : items)
+ {
+ if (item.getItemPath().equalsIgnoreCase(itemPath))
+ return item;
+ }
+ return null;
+ }
}
diff --git a/src/com/groksoft/els/repository/Repository.java b/src/com/groksoft/els/repository/Repository.java
index c21a9904..6d19a344 100644
--- a/src/com/groksoft/els/repository/Repository.java
+++ b/src/com/groksoft/els/repository/Repository.java
@@ -109,6 +109,13 @@ public void exportText() throws MungeException
}
}
+ /**
+ * Get the right-side item name.
+ *
+ * @param item The Item
+ * @return String of the right-side of the itemPath
+ * @throws MungeException
+ */
public String getItemName(Item item) throws MungeException
{
String path = item.getItemPath();
@@ -509,17 +516,22 @@ public void normalize() throws MungeException
*/
public String normalizePath(String toFlavor, String path) throws MungeException
{
- if (!toFlavor.equalsIgnoreCase(libraryData.libraries.flavor))
- {
- String to = Utils.getFileSeparator(toFlavor);
- path = normalizeSubst(path, Utils.getFileSeparator(libraryData.libraries.flavor), to);
- }
+ String to = Utils.getFileSeparator(toFlavor);
+ path = normalizeSubst(path, Utils.getFileSeparator(libraryData.libraries.flavor), to);
return path;
}
+ /**
+ * Normalize a path with a specific path separator character.
+ *
+ * @param path The path to normalize
+ * @param from The previous path separator character
+ * @param to The new path separator character
+ * @return String normalized path
+ */
private String normalizeSubst(String path, String from, String to)
{
- return path.replaceAll(from, to);
+ return path.replaceAll(from, to).replaceAll("\\|", to);
}
/**
@@ -725,6 +737,10 @@ private int scanDirectory(Library library, String base, String directory) throws
isSym = Files.isSymbolicLink(path); // is symbolic link check
item.setSymLink(isSym);
item.setLibrary(library.name); // the library name
+ if (!Utils.isFileOnly(item.getItemPath()))
+ {
+ item.setItemSubdirectory(Utils.pipe(this, Utils.getLeftPath(item.getItemPath(), getSeparator())));
+ }
library.items.add(item);
if (isDir)
@@ -757,13 +773,13 @@ private void scanSources(Library lib) throws MungeException
lib.items = null;
for (String src : lib.sources)
{
- logger.info(" " + src);
+ logger.debug(" " + src);
scanDirectory(lib, src, src);
}
}
/**
- * Sort collection.
+ * Sort a specific library's collection.
*/
public void sort(Library lib)
{
@@ -849,7 +865,7 @@ public void validate() throws Exception
throw new MungeException("libraries.bibliography must be defined");
}
- logger.info("Validating Libraries " + getJsonFilename());
+ logger.info("Validating " + lbs.description + " Libraries in:+ " + getJsonFilename());
for (int i = 0; i < lbs.bibliography.length; i++)
{
Library lib = lbs.bibliography[i];
@@ -866,7 +882,7 @@ public void validate() throws Exception
if ((!cfg.isSpecificLibrary() || cfg.isSelectedLibrary(lib.name)) &&
(!cfg.isSpecificExclude() || !cfg.isExcludedLibrary(lib.name)))
{
- logger.info(" library: " + lib.name +
+ logger.debug(" library: " + lib.name +
", " + lib.sources.length + " sources" +
(lib.items != null && lib.items.size() > 0 ? ", " + lib.items.size() + " items" : ""));
// validate sources paths
@@ -880,7 +896,7 @@ public void validate() throws Exception
{
throw new MungeException("bibliography[" + i + "].sources[" + j + "]: " + lib.sources[j] + " does not exist");
}
- logger.info(" src: " + lib.sources[j]);
+ logger.debug(" src: " + lib.sources[j]);
// validate item path
if (lib.items != null && lib.items.size() > 0)
diff --git a/src/com/groksoft/els/sftp/ClientSftp.java b/src/com/groksoft/els/sftp/ClientSftp.java
index 8240fd7a..e6e9e95f 100755
--- a/src/com/groksoft/els/sftp/ClientSftp.java
+++ b/src/com/groksoft/els/sftp/ClientSftp.java
@@ -144,6 +144,7 @@ public boolean startClient()
*/
public void stopClient()
{
+ logger.debug("Disconnecting sftp: " + (hostname == null ? "localhost" : hostname) + ":" + hostport);
if (jChannel != null)
jChannel.disconnect();
diff --git a/src/com/groksoft/els/sftp/EventListener.java b/src/com/groksoft/els/sftp/EventListener.java
index deea058b..1132801d 100644
--- a/src/com/groksoft/els/sftp/EventListener.java
+++ b/src/com/groksoft/els/sftp/EventListener.java
@@ -16,7 +16,7 @@
/**
* Apache Mina sftp event listener
- *
+ *
* See: https://mina.apache.org/sshd-project/apidocs/org/apache/sshd/server/subsystem/sftp/SftpEventListener.html
*/
diff --git a/src/com/groksoft/els/sftp/ServeSftp.java b/src/com/groksoft/els/sftp/ServeSftp.java
index fb2b35ae..569aaf32 100755
--- a/src/com/groksoft/els/sftp/ServeSftp.java
+++ b/src/com/groksoft/els/sftp/ServeSftp.java
@@ -19,6 +19,7 @@
import java.net.SocketAddress;
import java.security.PublicKey;
import java.util.Collections;
+import java.util.Random;
import java.util.Set;
/*
@@ -34,12 +35,11 @@
public class ServeSftp implements SftpErrorStatusDataHandler
{
- private transient Logger logger = LogManager.getLogger("applog");
-
private String hostname;
private int listenport;
- private int loginAttempts = 1;
+ private transient Logger logger = LogManager.getLogger("applog");
private String loginAttemptAddress = "";
+ private int loginAttempts = 1;
private Repository myRepo;
private String password;
private SshServer sshd;
@@ -69,6 +69,24 @@ public ServeSftp(Repository mine, Repository theirs, boolean primaryServers)
password = myRepo.getLibraryData().libraries.key;
}
+ /**
+ * Get a formatted String of bound IP addresses for this session
+ *
+ * @return
+ */
+ private String getIps()
+ {
+ // assemble listen IP(s)
+ String ips = "";
+ Set addrs;
+ addrs = sshd.getBoundAddresses();
+ for (SocketAddress a : addrs)
+ {
+ ips = ips + a.toString() + " ";
+ }
+ return ips;
+ }
+
@Override
public String resolveErrorMessage(SftpSubsystemEnvironment sftpSubsystem, int id, Throwable e, int subStatus, int cmd, Object... args)
{
@@ -83,6 +101,9 @@ public int resolveSubStatus(SftpSubsystemEnvironment sftpSubsystem, int id, Thro
return 1;
}
+ /**
+ * Start this SFTP server session
+ */
public void startServer()
{
try
@@ -129,7 +150,7 @@ public boolean authenticate(String s, String s1, ServerSession serverSession) th
authenticated = true;
loginAttempts = 1;
loginAttemptAddress = "";
- logger.info("ServeSftp server connected to " + serverSession.getClientAddress().toString());
+ logger.info("Sftp server connected to: " + serverSession.getClientAddress().toString());
}
else
{
@@ -142,10 +163,18 @@ public boolean authenticate(String s, String s1, ServerSession serverSession) th
loginAttempts = 1;
}
loginAttemptAddress = serverSession.getClientAddress().toString();
- logger.warn("Sftp login attempt " + loginAttempts + " failed using \"" + user + "\n/\"" + password + "\n from " + serverSession.getClientAddress());
+ logger.warn("Sftp login attempt " + loginAttempts + " failed, user \"" + user + "\n/\"" + password + "\n from " + serverSession.getClientAddress());
if (loginAttempts > 3)
{
- // todo Random sleep, 1 to 3 minutes
+ try
+ {
+ // random sleep for 1-3 minutes to discourage automated attacks
+ Random rand = new Random();
+ Thread.sleep(rand.nextInt(3) * 1000L);
+ }
+ catch (InterruptedException e)
+ {
+ }
}
}
return authenticated;
@@ -155,27 +184,25 @@ public boolean authenticate(String s, String s1, ServerSession serverSession) th
sshd.start();
// assemble listen IP(s)
- String ips = "";
- Set addrs;
- addrs = sshd.getBoundAddresses();
- for (SocketAddress a : addrs)
- {
- ips = ips + a.toString() + " ";
- }
- logger.info("ServeSftp server is listening on: " + ips);
+ String ips = getIps();
+ logger.info("Sftp server is listening on: " + ips);
}
catch (IOException e)
{
e.printStackTrace();
- logger.info("ServeSftp server cannot start secure channel");
+ logger.warn("Sftp server cannot start secure channel");
}
}
+ /**
+ * Stop this SFTP session
+ */
public void stopServer()
{
try
{
- logger.info("ServeSftp server listener stopping");
+ String ips = getIps();
+ logger.debug("Stopping sftp server on: " + ips);
sshd.stop();
}
catch (Exception e)
diff --git a/src/com/groksoft/els/storage/Storage.java b/src/com/groksoft/els/storage/Storage.java
index de8de774..813b1b1f 100755
--- a/src/com/groksoft/els/storage/Storage.java
+++ b/src/com/groksoft/els/storage/Storage.java
@@ -1,37 +1,52 @@
package com.groksoft.els.storage;
-import java.io.*;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-// see https://github.com/google/gson
import com.google.gson.Gson;
-
-// see https://logging.apache.org/log4j/2.x/
+import com.groksoft.els.MungeException;
+import com.groksoft.els.Utils;
import com.groksoft.els.repository.Libraries;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import com.groksoft.els.MungeException;
-import com.groksoft.els.Utils;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
/**
* The type Storage.
*/
public class Storage
{
+ public static final long MINIMUM_BYTES = 1073741824L; // minimum minimum bytes (1GB)
+ private String jsonFilename = "";
private transient Logger logger = LogManager.getLogger("applog");
-
// TargetData members
private TargetData targetData = null;
- private String jsonFilename = "";
-
- public static final long MINIMUM_BYTES = 1073741824L; // minimum minimum bytes (1GB)
/**
* Instantiates a new Storage instance.
*/
- public Storage() {
+ public Storage()
+ {
+ }
+
+ /**
+ * Gets Storage filename.
+ *
+ * @return the TargetData filename
+ */
+ public String getJsonFilename()
+ {
+ return jsonFilename;
+ }
+
+ /**
+ * Sets Storage file.
+ *
+ * @param jsonFilename the TargetData file
+ */
+ public void setJsonFilename(String jsonFilename)
+ {
+ this.jsonFilename = jsonFilename;
}
/**
@@ -66,7 +81,6 @@ public Target getLibraryTarget(String libraryName) throws MungeException
/**
* Normalize target paths based on "flavor"
- *
*/
private void normalize(String flavor)
{
@@ -109,7 +123,8 @@ private void normalize(String flavor)
*/
public void read(String filename, String flavor) throws MungeException
{
- try {
+ try
+ {
String json;
Gson gson = new Gson();
logger.info("Reading Targets file " + filename);
@@ -117,7 +132,9 @@ public void read(String filename, String flavor) throws MungeException
json = new String(Files.readAllBytes(Paths.get(filename)));
targetData = gson.fromJson(json, TargetData.class);
normalize(flavor);
- } catch (IOException ioe) {
+ }
+ catch (IOException ioe)
+ {
throw new MungeException("Exception while reading targets: " + filename + " trace: " + Utils.getStackTrace(ioe));
}
}
@@ -131,60 +148,52 @@ public void validate() throws MungeException
{
long minimumSize;
- if (targetData == null) {
+ if (targetData == null)
+ {
throw new MungeException("TargetData are null");
}
Targets targets = targetData.targets;
- if (targets.description == null || targets.description.length() == 0) {
+ if (targets.description == null || targets.description.length() == 0)
+ {
throw new MungeException("targets.description must be defined");
}
- for (int i = 0; i < targets.storage.length; ++i) {
+ for (int i = 0; i < targets.storage.length; ++i)
+ {
Target t = targets.storage[i];
- if (t.name == null || t.name.length() == 0) {
+ if (t.name == null || t.name.length() == 0)
+ {
throw new MungeException("storage.name [" + i + "] must be defined");
}
- if (t.minimum == null || t.minimum.length() == 0) {
+ if (t.minimum == null || t.minimum.length() == 0)
+ {
throw new MungeException("storage.minimum [" + i + "] must be defined");
}
long min = Utils.getScaledValue(t.minimum);
- if (min < MINIMUM_BYTES) { // non-fatal warning
+ if (min < MINIMUM_BYTES)
+ { // non-fatal warning
logger.warn("Storage.minimum [" + i + "] " + t.name + " of " + t.minimum + " is less than allowed minimum of " + (MINIMUM_BYTES / 1024 / 1024) + "MB. Using allowed minimum.");
}
- if (t.locations == null || t.locations.length == 0) {
+ if (t.locations == null || t.locations.length == 0)
+ {
throw new MungeException("storage.locations [" + i + "] " + t.name + " must be defined");
}
- for (int j = 0; j < t.locations.length; ++j) {
- if (t.locations[j].length() == 0) {
+ for (int j = 0; j < t.locations.length; ++j)
+ {
+ if (t.locations[j].length() == 0)
+ {
throw new MungeException("storage[" + i + "].locations[" + j + "] " + t.name + " must be defined");
}
- if (Files.notExists(Paths.get(t.locations[j]))) {
+ if (Files.notExists(Paths.get(t.locations[j])))
+ {
throw new MungeException("storage[" + i + "].locations[" + j + "]: " + t.locations[j] + " does not exist");
}
logger.debug(" loc: " + t.locations[j]);
}
}
- logger.info("Targets validation successful: " + getJsonFilename());
- }
-
- /**
- * Gets Storage filename.
- *
- * @return the TargetData filename
- */
- public String getJsonFilename() {
- return jsonFilename;
- }
-
- /**
- * Sets Storage file.
- *
- * @param jsonFilename the TargetData file
- */
- public void setJsonFilename(String jsonFilename) {
- this.jsonFilename = jsonFilename;
+ logger.debug("Targets validation successful: " + getJsonFilename());
}
}
diff --git a/src/com/groksoft/els/storage/Target.java b/src/com/groksoft/els/storage/Target.java
index ad9fbaad..ea01a2b5 100644
--- a/src/com/groksoft/els/storage/Target.java
+++ b/src/com/groksoft/els/storage/Target.java
@@ -6,17 +6,15 @@
public class Target
{
/**
- * The target Name.
+ * The Locations.
*/
- public String name;
-
+ public String[] locations;
/**
* The Minimum space available limit.
*/
public String minimum;
-
/**
- * The Locations.
+ * The target Name.
*/
- public String[] locations;
+ public String name;
}
diff --git a/src/com/groksoft/els/stty/ClientStty.java b/src/com/groksoft/els/stty/ClientStty.java
index f2146b4a..814d71a5 100755
--- a/src/com/groksoft/els/stty/ClientStty.java
+++ b/src/com/groksoft/els/stty/ClientStty.java
@@ -1,10 +1,11 @@
package com.groksoft.els.stty;
import com.groksoft.els.Configuration;
+import com.groksoft.els.Context;
import com.groksoft.els.MungeException;
import com.groksoft.els.Utils;
-import com.groksoft.els.stty.gui.TerminalGui;
import com.groksoft.els.repository.Repository;
+import com.groksoft.els.stty.gui.TerminalGui;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -21,27 +22,24 @@
*/
public class ClientStty
{
- private transient Logger logger = LogManager.getLogger("applog");
-
+ TerminalGui gui = null;
+ DataInputStream in = null;
+ DataOutputStream out = null;
private Configuration cfg;
private boolean isConnected = false;
private boolean isTerminal = false;
- private Socket socket;
-
- DataInputStream in = null;
- DataOutputStream out = null;
- TerminalGui gui = null;
-
- private Repository myRepo;
- private Repository theirRepo;
+ private transient Logger logger = LogManager.getLogger("applog");
private String myKey;
- private String theirKey;
+ private Repository myRepo;
private boolean primaryServers;
+ private Socket socket;
+ private String theirKey;
+ private Repository theirRepo;
/**
* Instantiate a ClientStty.
*
- * @param config The Configuration object
+ * @param config The Configuration object
* @param isManualTerminal True if an interactive client, false if an automated client
*/
public ClientStty(Configuration config, boolean isManualTerminal, boolean primaryServers)
@@ -51,6 +49,13 @@ public ClientStty(Configuration config, boolean isManualTerminal, boolean primar
this.primaryServers = primaryServers;
}
+ /**
+ * Return available space disk of the remote location
+ *
+ * @param location Path on remote
+ * @return long Available space in bytes
+ * @throws Exception
+ */
public long availableSpace(String location) throws Exception
{
long space = 0L;
@@ -63,6 +68,16 @@ public long availableSpace(String location) throws Exception
return space;
}
+ /**
+ * Read opening terminal banner for possible commands
+ *
+ * Handles subscriber-side commands sent to publisher at login time
+ * for RequestCollection to retrieve the current subscriber collection,
+ * and RequestTargets to retrieve the current subscriber targets.
+ *
+ * @return true if commands were present and processed
+ * @throws Exception
+ */
public boolean checkBannerCommands() throws Exception
{
boolean hasCommands = false;
@@ -106,6 +121,14 @@ else if (cmdSplit[i].equals("RequestTargets"))
return hasCommands;
}
+ /**
+ * Connect this STTY to the other end
+ *
+ * @param mine Local Repository
+ * @param theirs Remote Repository
+ * @return true if connected
+ * @throws Exception
+ */
public boolean connect(Repository mine, Repository theirs) throws Exception
{
this.myRepo = mine;
@@ -132,7 +155,7 @@ public boolean connect(Repository mine, Repository theirs) throws Exception
this.socket = new Socket(host, port);
in = new DataInputStream(socket.getInputStream());
out = new DataOutputStream(socket.getOutputStream());
- logger.info("Successfully connected to: " + this.socket.getInetAddress().toString());
+ logger.info("Successfully connected stty to: " + Utils.formatAddresses(this.socket));
}
catch (Exception e)
{
@@ -147,32 +170,47 @@ public boolean connect(Repository mine, Repository theirs) throws Exception
}
else
{
- logger.error("Connection to " + host + ":" + port + " failed handshake");
+ logger.error("Connection to " + Utils.formatAddresses(socket) + " failed handshake");
}
}
}
else
{
- throw new MungeException("cannot get site from -r specified remote subscriber library");
+ throw new MungeException("Cannot get site from -s | -S specified remote subscriber library");
}
return isConnected;
}
+ /**
+ * Disconnect this STTY from the other end
+ */
public void disconnect()
{
try
{
- gui.stop();
- out.flush();
- out.close();
- in.close();
+ if (isConnected)
+ {
+ isConnected = false;
+ logger.debug("Disconnecting stty: " + Utils.formatAddresses(socket));
+ if (gui != null)
+ gui.stop();
+ out.flush();
+ out.close();
+ in.close();
+ }
}
catch (Exception e)
{
}
}
+ /**
+ * Start an interactive GUI terminal session
+ *
+ * @return
+ * @throws Exception
+ */
public int guiSession() throws Exception
{
int returnValue = 0;
@@ -181,6 +219,12 @@ public int guiSession() throws Exception
return returnValue;
}
+ /**
+ * Perform a handshake with the other end
+ *
+ * @return true if connection authenticated
+ * @throws Exception
+ */
private boolean handshake() throws Exception
{
boolean valid = false;
@@ -213,21 +257,77 @@ private boolean handshake() throws Exception
// ignore
}
}
+ else if (input.equalsIgnoreCase("Terminal session not allowed"))
+ logger.warn("Attempted to login interactively but terminal sessions are not allowed");
}
return valid;
}
+ /**
+ * Return if this STTY session is connected
+ *
+ * @return
+ */
public boolean isConnected()
{
return isConnected;
}
+ /**
+ * Send command to Hint Status Server to quit
+ *
+ * @param context The Context
+ * @param fault Pass-through fault indicator
+ * @return Resulting fault indicator
+ */
+ public boolean quitStatusServer(Context context, boolean fault)
+ {
+ if (cfg.isQuitStatusServer())
+ {
+ if (context.statusRepo == null)
+ {
+ logger.warn("-q requires a -h hints file");
+ return true;
+ }
+ try
+ {
+ if (isConnected())
+ {
+ logger.info("Sending quit command to hint status server: " + context.statusRepo.getLibraryData().libraries.description);
+ context.statusStty.send("quit");
+ }
+ else
+ logger.warn("Could not send quit command to hint status server: " + context.statusRepo.getLibraryData().libraries.description);
+ }
+ catch (Exception e)
+ {
+ logger.error(Utils.getStackTrace(e));
+ fault = true;
+ }
+ }
+ return fault;
+ }
+
+ /**
+ * Receive a response from the other end
+ *
+ * @return String of response text
+ * @throws Exception
+ */
public String receive() throws Exception
{
String response = Utils.readStream(in, theirRepo.getLibraryData().libraries.key);
return response;
}
+ /**
+ * Retrieve remote data and store it in a file
+ *
+ * @param filename File path to store the data
+ * @param command The command to send
+ * @return The resulting date-stamped file path
+ * @throws Exception
+ */
public String retrieveRemoteData(String filename, String command) throws Exception
{
String location = null;
@@ -239,6 +339,9 @@ public String retrieveRemoteData(String filename, String command) throws Excepti
DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");
LocalDateTime now = LocalDateTime.now();
String stamp = dtf.format(now);
+
+ // TODO Make better paths for the temporary -received- files
+
location = filename + "_" + command + "-received-" + stamp + ".json";
try
{
@@ -254,6 +357,13 @@ public String retrieveRemoteData(String filename, String command) throws Excepti
return location;
}
+ /**
+ * Make a round-trip to the other end by sending a command and receiving the response
+ *
+ * @param command The command to send
+ * @return String of the response
+ * @throws Exception
+ */
public String roundTrip(String command) throws Exception
{
send(command);
@@ -261,6 +371,12 @@ public String roundTrip(String command) throws Exception
return response;
}
+ /**
+ * Send a command to the other end
+ *
+ * @param command The command to send
+ * @throws Exception
+ */
public void send(String command) throws Exception
{
Utils.writeStream(out, theirRepo.getLibraryData().libraries.key, command);
diff --git a/src/com/groksoft/els/stty/Connection.java b/src/com/groksoft/els/stty/Connection.java
index c951fc72..8ad4aa62 100644
--- a/src/com/groksoft/els/stty/Connection.java
+++ b/src/com/groksoft/els/stty/Connection.java
@@ -1,14 +1,14 @@
package com.groksoft.els.stty;
+import com.groksoft.els.Utils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.io.*;
-import java.net.*;
+import java.net.Socket;
/**
* Handle individual client connections.
- *
+ *
* The Connection class is a subclass of Thread. It handles individual
* connections between a Service and a client user. Each connection has a
* separate thread. Each Service can have multiple connection requests pending
@@ -16,73 +16,83 @@
*/
public class Connection extends Thread
{
- protected static Logger logger = LogManager.getLogger("applog");//
+ protected static Logger logger = LogManager.getLogger("applog");//
- /** The service for the connection */
- protected DaemonBase service;
- /** The socket for the connection */
- protected Socket socket;
+ /**
+ * The service for the connection
+ */
+ protected DaemonBase service;
+ /**
+ * The socket for the connection
+ */
+ protected Socket socket;
- /**
- * Constructor.
- *
- * Connection objects are created by Listener threads as part of the
- * server's thread group. The superclass constructor is called to create a
- * new thread to handle the connection request.
- *
- * @param aSocket Socket for connection
- * @param aService Service for connection
- */
- public Connection (Socket aSocket, DaemonBase aService)
- {
- super("Daemon.Connection:" + aSocket.getInetAddress().getHostAddress() + ":" + aSocket.getPort());
- this.socket = aSocket;
- this.service = aService;
- } // constructor
-
- /**
- * Get the associated Daemon instance
- */
- public DaemonBase getConsole ()
- {
- return service;
- }
-
- /**
- * Get the associated Socket instance
- */
- public Socket getSocket ()
- {
- return socket;
- }
+ /**
+ * Constructor.
+ *
+ * Connection objects are created by Listener threads as part of the
+ * server's thread group. The superclass constructor is called to create a
+ * new thread to handle the connection request.
+ *
+ * @param aSocket Socket for connection
+ * @param aService Service for connection
+ */
+ public Connection(Socket aSocket, DaemonBase aService)
+ {
+ super("Daemon.Connection:" + Utils.formatAddresses(aSocket));
+ this.socket = aSocket;
+ this.service = aService;
+ } // constructor
- /**
- * Run the service for this connection.
- *
- * Creates input and output streams for the connection and calls the
- * interface method for the Service. Calls endConnection() when
- * the method returns for any reason.
- *
- */
- public void run ()
- {
- try
- {
- service.process(socket);
- }
- catch (Exception e)
- {
- logger.info(e);
- }
- finally
- {
- // notify the ConnectionManager that this connection has closed
- ServeStty cm = ServeStty.getInstance();
- if (cm != null)
- {
- cm.endConnection();
- }
- }
- }
-} // Connection
+ /**
+ * Get the associated Daemon instance
+ */
+ public DaemonBase getConsole()
+ {
+ return service;
+ }
+
+ /**
+ * Get the associated Socket instance
+ */
+ public Socket getSocket()
+ {
+ return socket;
+ }
+ /**
+ * Run the service for this connection.
+ *
+ * Creates input and output streams for the connection and calls the
+ * interface method for the Service. Calls endConnection() when
+ * the method returns for any reason.
+ */
+ public void run()
+ {
+ boolean stop = false;
+ try
+ {
+ stop = service.process(socket);
+ }
+ catch (Exception e)
+ {
+ logger.info(e);
+ stop = true;
+ }
+ finally
+ {
+ // notify the ConnectionManager that this connection has closed
+ logger.debug("Closing stty connection to: " + Utils.formatAddresses(socket));
+ ServeStty cm = ServeStty.getInstance();
+ if (cm != null)
+ {
+ cm.endConnection();
+ if (stop)
+ {
+ cm.stopServer();
+ }
+ }
+ }
+ }
+
+} // Connection
diff --git a/src/com/groksoft/els/stty/DaemonBase.java b/src/com/groksoft/els/stty/DaemonBase.java
index 76c5d608..e6e401ea 100644
--- a/src/com/groksoft/els/stty/DaemonBase.java
+++ b/src/com/groksoft/els/stty/DaemonBase.java
@@ -1,16 +1,21 @@
package com.groksoft.els.stty;
import com.groksoft.els.Configuration;
+import com.groksoft.els.Utils;
import com.groksoft.els.repository.Repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.io.*;
-import java.net.*;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.InetAddress;
+import java.net.Socket;
/**
* Daemon service.
- *
+ *
* The Daemon service is the command interface used to communicate between
* the endpoints.
*/
@@ -20,20 +25,18 @@ public abstract class DaemonBase
protected InetAddress address;
protected boolean authorized = false;
+ protected Configuration cfg;
protected boolean connected = false;
- protected int port;
- protected Socket socket;
- protected boolean stop = false;
-
protected DataInputStream in = null;
+ protected String myKey;
+ protected Repository myRepo;
protected DataOutputStream out = null;
+ protected int port;
protected String response = "";
-
- protected Configuration cfg;
- protected Repository myRepo;
- protected String myKey;
- protected Repository theirRepo;
+ protected Socket socket;
+ protected boolean stop = false;
protected String theirKey;
+ protected Repository theirRepo;
/**
* Instantiate the Daemon service
@@ -42,8 +45,11 @@ public DaemonBase(Configuration config, Repository mine, Repository theirs)
{
this.cfg = config;
this.myRepo = mine;
- this.theirRepo = theirs;
- this.theirKey = theirRepo.getLibraryData().libraries.key;
+ if (theirs != null)
+ {
+ this.theirRepo = theirs;
+ this.theirKey = this.theirRepo.getLibraryData().libraries.key;
+ }
this.myKey = myRepo.getLibraryData().libraries.key;
} // constructor
@@ -52,7 +58,7 @@ public DaemonBase(Configuration config, Repository mine, Repository theirs)
*
* @param aWriter The PrintWriter to be used to print the list.
*/
- public synchronized void dumpStatistics (PrintWriter aWriter)
+ public synchronized void dumpStatistics(PrintWriter aWriter)
{
/*
aWriter.println("\r\Daemon currently connected: " + ((connected) ? "true" : "false"));
@@ -75,30 +81,33 @@ public synchronized void dumpStatistics (PrintWriter aWriter)
*
* @return Short name of this service.
*/
- public String getName ()
+ public String getName()
{
return "DaemonBase";
}
- /**
- * Request the Daemon service to stop
- */
- public void requestStop ()
+ public Socket getSocket()
{
- this.stop = true;
- logger.info("Requesting stop for session on port " + socket.getPort() + " to " + socket.getInetAddress());
+ return socket;
}
+ /**
+ * Perform initial handshake for this session.
+ */
+ public abstract String handshake();
+
/**
* Process a connection request to the Daemon service.
- *
*/
- public abstract void process(Socket aSocket) throws IOException, Exception;
+ public abstract boolean process(Socket aSocket) throws IOException, Exception;
/**
- * Perform initial handshake for this session.
- *
+ * Request the Daemon service to stop
*/
- public abstract boolean handshake();
+ public void requestStop()
+ {
+ this.stop = true;
+ logger.debug("Requesting stop for stty session: " + Utils.formatAddresses(socket));
+ }
} // DaemonBase
diff --git a/src/com/groksoft/els/stty/Listener.java b/src/com/groksoft/els/stty/Listener.java
index 3a66f562..82c1b0ab 100755
--- a/src/com/groksoft/els/stty/Listener.java
+++ b/src/com/groksoft/els/stty/Listener.java
@@ -5,115 +5,120 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.io.*;
+import java.io.IOException;
+import java.io.InterruptedIOException;
import java.net.*;
/**
* Listen for a connection request for a service.
- *
+ *
* The Listener class is a subclass of Thread. It listens for connections
* on a specified port. When a connection is requested it is handled by the
* ConnectionManager. There is one Listener for each service on a specified
* port.
- *
*/
public class Listener extends Thread
{
- protected static Logger logger = LogManager.getLogger("applog");//
+ /**
+ * The default timeout for socket connections, in milliseconds
+ */
+ protected static final int socketTimeout = 2000;
+ protected static Logger logger = LogManager.getLogger("applog");//
+ private InetAddress addr;
+ private Configuration cfg;
+ /**
+ * The socket to listen on for the associated service
+ */
+ private ServerSocket listenSocket;
+ /**
+ * The port to listen on for the associated service
+ */
+ private int port;
+ /**
+ * Flag used to determine when to stop listening
+ */
+ private boolean stop = false;
- /** The default timeout for socket connections, in milliseconds */
- protected static final int socketTimeout = 2000;
-
- /** The socket to listen on for the associated service */
- private ServerSocket listenSocket;
- /** The port to listen on for the associated service */
- private int port;
- /** Flag used to determine when to stop listening */
- private boolean stop = false;
-
- private Configuration cfg;
- private InetAddress addr;
-
- /**
- * Setup a new Listener on a specified port.
- *
- * The socket is set with a timeout to allow accept() to be interrupted, and
- * the service to be removed from the server.
- *
- * @param group The thread group used for the listener.
- * @param aPort The port to listen on.
- */
- public Listener (ThreadGroup group, String host, int aPort, Configuration config) throws Exception
- {
- super(group, "Listener:" + host + ":" + aPort);
+ /**
+ * Setup a new Listener on a specified port.
+ *
+ * The socket is set with a timeout to allow accept() to be interrupted, and
+ * the service to be removed from the server.
+ *
+ * @param group The thread group used for the listener.
+ * @param aPort The port to listen on.
+ */
+ public Listener(ThreadGroup group, String host, int aPort, Configuration config) throws Exception
+ {
+ super(group, "Listener:" + host + ":" + aPort);
- // setup this listener
+ // setup this listener
this.cfg = config;
- this.port = aPort;
- addr = Inet4Address.getByName(host);
+ this.port = aPort;
+ addr = Inet4Address.getByName(host);
- listenSocket = new ServerSocket(this.port, 5, addr);
+ listenSocket = new ServerSocket(this.port, 5, addr);
- // set a non-zero timeout on the socket so accept() may be interrupted
- listenSocket.setSoTimeout(socketTimeout);
- } // constructor
+ // set a non-zero timeout on the socket so accept() may be interrupted
+ listenSocket.setSoTimeout(socketTimeout);
+ } // constructor
public String getInetAddr()
{
return addr.getHostAddress();
}
- /**
- * Politely request the listener to stop.
- *
- */
- public void requestStop ()
- {
- this.stop = true;
- this.interrupt();
- }
+ /**
+ * Politely request the listener to stop.
+ */
+ public void requestStop()
+ {
+ this.stop = true;
+ this.interrupt();
+ }
- /**
- * Run listen thread body.
- *
- * Waits for a connection request. When a request is received an attempt is
- * made to create a new connection thread via a call to the addConnection()
- * method.
- */
- public void run ()
- {
- while (stop == false)
- {
- try
- {
- Socket theSocket = (Socket) listenSocket.accept();
- theSocket.setTcpNoDelay(true);
+ /**
+ * Run listen thread body.
+ *
+ * Waits for a connection request. When a request is received an attempt is
+ * made to create a new connection thread via a call to the addConnection()
+ * method.
+ */
+ public void run()
+ {
+ Socket socket = null;
+ while (stop == false)
+ {
+ try
+ {
+ socket = (Socket) listenSocket.accept();
+ socket.setTcpNoDelay(true);
//theSocket.setSoLinger(false, -1);
- theSocket.setSoLinger(true, 10000); // linger 10 seconds after transmission completed
+ socket.setSoLinger(true, 10000); // linger 10 seconds after transmission completed
- ServeStty.getInstance().addConnection(theSocket);
- }
- catch (SocketTimeoutException e)
- {
- //logger.info("Listen accept timeout on port " + port + ", stop=" + ((stop)?"true, stopping":"false, continuing"));
- continue;
- }
- catch (InterruptedIOException e)
- {
- logger.info("listener interrupted on port " + port + ", stop=" + ((stop)?"true":"false"));
- }
- catch (IOException e)
- {
- logger.error(e);
- stop = true;
- }
- catch (MungeException e)
- {
- logger.error(e);
- stop = true;
- }
- }
- if (logger != null)
- logger.info("Stopped listener on port " + port);
- }
+ ServeStty.getInstance().addConnection(socket);
+ }
+ catch (SocketTimeoutException e)
+ {
+ //logger.info("Listen accept timeout on port " + port + ", stop=" + ((stop)?"true, stopping":"false, continuing"));
+ continue;
+ }
+ catch (InterruptedIOException e)
+ {
+ logger.debug("Listener interrupted on port " + port + ", stop=" + ((stop) ? "true" : "false"));
+ }
+ catch (IOException e)
+ {
+ logger.error(e);
+ stop = true;
+ }
+ catch (MungeException e)
+ {
+ logger.error(e);
+ stop = true;
+ }
+ }
+ if (logger != null && socket != null)
+ logger.debug("Stopping stty listener on: " + socket.getInetAddress().toString() + ":" + socket.getPort());
+ }
} // Listener
diff --git a/src/com/groksoft/els/stty/ServeStty.java b/src/com/groksoft/els/stty/ServeStty.java
index 4efa3cee..12772fbd 100755
--- a/src/com/groksoft/els/stty/ServeStty.java
+++ b/src/com/groksoft/els/stty/ServeStty.java
@@ -1,13 +1,19 @@
package com.groksoft.els.stty;
-import com.groksoft.els.*;
+import com.groksoft.els.Configuration;
+import com.groksoft.els.Context;
+import com.groksoft.els.MungeException;
+import com.groksoft.els.Utils;
import com.groksoft.els.repository.Repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.io.*;
-import java.net.*;
-import java.util.*;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.Socket;
+import java.util.Collection;
+import java.util.Hashtable;
+import java.util.Vector;
/**
* Manage all connections and enforce limits.
@@ -24,12 +30,6 @@
*/
public class ServeStty extends Thread
{
- private transient Logger logger = LogManager.getLogger("applog");
-
- /**
- * The list of all service connections
- */
- private Vector allConnections;
/**
* The single instance of this class
*/
@@ -38,22 +38,25 @@ public class ServeStty extends Thread
* The maximum connections allowed for this entire server instance
*/
protected int maxConnections;
- /**
- * Count of total connections since started
- */
- private int totalConnections = 0;
/**
* Flag used to determine when to stop listening
*/
private boolean _stop = false;
-
- private Hashtable allSessions;
+ /**
+ * The list of all service connections
+ */
+ private Vector allConnections;
private ThreadGroup allSessionThreads;
-
+ private Hashtable allSessions;
private Configuration cfg;
private Context context;
private int listenPort;
+ private transient Logger logger = LogManager.getLogger("applog");
private boolean primaryServers;
+ /**
+ * Count of total connections since started
+ */
+ private int totalConnections = 0;
/**
* Instantiates the ServeStty object and set it as a daemon so the Java
@@ -77,6 +80,14 @@ public ServeStty(ThreadGroup aGroup, int aMaxConnections, Configuration config,
this.allSessionThreads = aGroup;
} // constructor
+ /**
+ * Get this instance.
+ */
+ public static ServeStty getInstance()
+ {
+ return instance;
+ }
+
/**
* Add a connection for a service.
*
@@ -98,15 +109,17 @@ public synchronized void addConnection(Socket aSocket) throws MungeException
// log it
logger.info("Maximum connections (" + maxConnections + ") exceeded");
- logger.info("Connection refused from " + aSocket.getInetAddress().getHostAddress() + ":" + aSocket.getPort());
+ logger.info("Connection refused from: " + Utils.formatAddresses(aSocket));
// close the connection
aSocket.close();
- } catch (IOException e)
+ }
+ catch (IOException e)
{
logger.info(e);
}
- } else
+ }
+ else
// if limit has not been reached
{
// create a connection thread for this request
@@ -114,17 +127,23 @@ public synchronized void addConnection(Socket aSocket) throws MungeException
if (cfg.isPublisherListener())
{
theConnection = new Connection(aSocket, new com.groksoft.els.stty.publisher.Daemon(cfg, context, context.publisherRepo, context.subscriberRepo));
- } else if (cfg.isSubscriberListener() || cfg.isSubscriberTerminal())
+ }
+ else if (cfg.isSubscriberListener() || cfg.isSubscriberTerminal())
{
theConnection = new Connection(aSocket, new com.groksoft.els.stty.subscriber.Daemon(cfg, context, context.subscriberRepo, context.publisherRepo));
- } else
+ }
+ else if (cfg.isStatusServer())
+ {
+ theConnection = new Connection(aSocket, new com.groksoft.els.stty.hintServer.Daemon(cfg, context, context.statusRepo, null));
+ }
+ else
{
throw new MungeException("FATAL: Unknown connection type");
}
allConnections.add(theConnection);
// log it
- logger.info((cfg.isPublisherListener() ? "Publisher" : "Subscriber") + " daemon opened " + aSocket.getInetAddress().getHostAddress() + ":" + aSocket.getPort());
+ logger.info((cfg.isStatusServer() ? "Status Server" : (cfg.isPublisherListener() ? "Publisher" : "Subscriber")) + " daemon opened stty: " + Utils.formatAddresses(aSocket));
// start the connection thread
theConnection.start();
@@ -132,21 +151,45 @@ public synchronized void addConnection(Socket aSocket) throws MungeException
}
}
+ protected void addListener(String host, int aPort) throws Exception
+ {
+ //Integer key = new Integer(aPort); // hashtable key
+
+ // do not allow duplicate port assignments
+ if (allSessions.get("Listener:" + host + ":" + aPort) != null)
+ throw new IllegalArgumentException("Port " + aPort + " already in use");
+
+ // create a listener on the port
+ Listener listener = new Listener(allSessionThreads, host, aPort, cfg);
+
+ // put it in the hashtable
+ allSessions.put("Listener:" + host + ":" + aPort, listener);
+
+ // log it
+ logger.info("Stty server is listening on: " + (host == null ? "localhost" : listener.getInetAddr()) + ":" + aPort);
+
+ // fire it up
+ listener.start();
+ }
+
/**
- * Start a session listener
+ * Dump statistics of connections.
*/
- public void startListening(Repository listenerRepo) throws Exception
+ public synchronized String dumpStatistics()
{
- if (listenerRepo != null &&
- listenerRepo.getLibraryData() != null &&
- listenerRepo.getLibraryData().libraries != null &&
- listenerRepo.getLibraryData().libraries.listen != null)
- {
- startServer(listenerRepo.getLibraryData().libraries.listen);
- } else
+ String data = "Listening on: " + listenPort + "\r\n" +
+ "Active connections: " + allConnections.size() + "\r\n";
+ for (int index = 0; index < allConnections.size(); ++index)
{
- throw new MungeException("cannot get site from -r specified remote library");
+ Connection c = (Connection) allConnections.elementAt(index);
+ data += " " + c.service.getName() + " to " + Utils.formatAddresses(c.socket) + "\r\n";
}
+
+ // dump connection counts
+ data += " Total connections since started: " + totalConnections + "\r\n";
+ data += " Maximum allowed connections: " + maxConnections + "\r\n";
+
+ return data;
}
/**
@@ -164,14 +207,6 @@ public synchronized void endConnection()
this.notify();
}
- /**
- * Get this instance.
- */
- public static ServeStty getInstance()
- {
- return instance;
- }
-
/**
* Get the connections Vector
*/
@@ -180,34 +215,6 @@ public Vector getAllConnections()
return this.allConnections;
}
- /**
- * Set or change the maximum number of connections allowed for this server.
- */
- public synchronized void setMaxConnections(int aMax)
- {
- maxConnections = aMax;
- }
-
- /**
- * Dump statistics of connections.
- */
- public synchronized String dumpStatistics()
- {
- String data = "Listening on: " + listenPort + "\r\n" +
- "Active connections: " + allConnections.size() + "\r\n";
- for (int index = 0; index < allConnections.size(); ++index)
- {
- Connection c = (Connection) allConnections.elementAt(index);
- data += " " + c.service.getName() + " to " + c.socket.getInetAddress().getHostAddress() + ":" + c.socket.getPort() + "\r\n";
- }
-
- // dump connection counts
- data += " Total connections since started: " + totalConnections + "\r\n";
- data += " Maximum allowed connections: " + maxConnections + "\r\n";
-
- return data;
- }
-
/**
* Politely request the listener to stop.
*/
@@ -236,7 +243,7 @@ public void requestStop()
public void run()
{
// log it
- logger.info("Starting ServeStty server for up to " + maxConnections + " incoming connections");
+ logger.info("Starting stty server for up to " + maxConnections + " incoming connections");
while (_stop == false)
{
for (int index = 0; index < allConnections.size(); ++index)
@@ -246,7 +253,6 @@ public void run()
if (!c.isAlive())
{
allConnections.removeElementAt(index);
- logger.info(c.service.getName() + " closed " + c.socket.getInetAddress().getHostAddress() + ":" + c.socket.getPort() + " port " + c.socket.getLocalPort());
}
}
@@ -257,33 +263,54 @@ public void run()
{
this.wait();
}
- } catch (InterruptedException e)
+ }
+ catch (InterruptedException e)
{
- logger.info("ServeStty interrupted, stop=" + ((_stop) ? "true" : "false"));
+ logger.debug("Stty interrupted, stop=" + ((_stop) ? "true" : "false"));
+ _stop = true;
}
- } // while (true)
- logger.info("Stopped ServeStty");
+ }
+
+ // when this server ends disconnect and stop other services
+ // otherwise the threads will never stop
+ if (context.statusStty != null)
+ {
+ context.statusStty.quitStatusServer(context, false);
+ context.statusStty.disconnect();
+ context.statusStty = null;
+ }
+ if (context.serveSftp != null)
+ {
+ context.serveSftp.stopServer();
+ context.serveSftp = null;
+ }
+ logger.debug("Stopping stty server");
}
- protected void addListener(String host, int aPort) throws Exception
+ /**
+ * Set or change the maximum number of connections allowed for this server.
+ */
+ public synchronized void setMaxConnections(int aMax)
{
- //Integer key = new Integer(aPort); // hashtable key
-
- // do not allow duplicate port assignments
- if (allSessions.get("Listener:" + host + ":" + aPort) != null)
- throw new IllegalArgumentException("Port " + aPort + " already in use");
-
- // create a listener on the port
- Listener listener = new Listener(allSessionThreads, host, aPort, cfg);
-
- // put it in the hashtable
- allSessions.put("Listener:" + host + ":" + aPort, listener);
-
- // log it
- logger.info("ServeStty server is listening on: " + (host == null ? "localhost" : listener.getInetAddr()) + ":" + aPort);
+ maxConnections = aMax;
+ }
- // fire it up
- listener.start();
+ /**
+ * Start a session listener
+ */
+ public void startListening(Repository listenerRepo) throws Exception
+ {
+ if (listenerRepo != null &&
+ listenerRepo.getLibraryData() != null &&
+ listenerRepo.getLibraryData().libraries != null &&
+ listenerRepo.getLibraryData().libraries.listen != null)
+ {
+ startServer(listenerRepo.getLibraryData().libraries.listen);
+ }
+ else
+ {
+ throw new MungeException("cannot get site from -r specified remote library");
+ }
}
public void startServer(String listen) throws Exception
@@ -302,7 +329,7 @@ public void startServer(String listen) throws Exception
}
if (listenPort < 1)
{
- logger.info("ServeStty is disabled");
+ logger.info("Stty is disabled");
}
}
@@ -310,24 +337,21 @@ public void stopServer()
{
if (allSessions != null)
{
- logger.info("Stopping all Sessions");
- Enumeration keys = allSessions.keys();
- while (keys.hasMoreElements())
+ logger.debug("Stopping all stty listener threads");
+ Collection lc = allSessions.values();
+ for (Listener listener : lc)
{
- //Integer port = (Integer)keys.nextElement();
- Connection conn = (Connection) keys.nextElement();
- Socket sock = conn.getSocket();
- Integer port = sock.getPort();
- Listener listener = (Listener) allSessions.get(port);
if (listener != null)
{
- listener.requestStop();
+ if (listener.isAlive())
+ listener.requestStop();
}
}
this.requestStop();
- } else
+ }
+ else
{
- logger.info("nothing to stop");
+ logger.debug("nothing to stop");
}
}
diff --git a/src/com/groksoft/els/stty/gui/TerminalGui.java b/src/com/groksoft/els/stty/gui/TerminalGui.java
index 5951658c..16c88ee5 100755
--- a/src/com/groksoft/els/stty/gui/TerminalGui.java
+++ b/src/com/groksoft/els/stty/gui/TerminalGui.java
@@ -102,7 +102,7 @@ public void actionPerformed(ActionEvent e)
private int build()
{
- frame = new JFrame("ELS " + cfg.getProgramVersionN() + " connected to " + theirRepo.getLibraryData().libraries.description);
+ frame = new JFrame("ELS " + cfg.getProgramVersion() + " connected to " + theirRepo.getLibraryData().libraries.description);
// try
// {
// UIManager.setLookAndFeel("com.sun.java.swing.plaf.windows.WindowsLookAndFeel");
diff --git a/src/com/groksoft/els/stty/hintServer/Daemon.java b/src/com/groksoft/els/stty/hintServer/Daemon.java
new file mode 100755
index 00000000..62448282
--- /dev/null
+++ b/src/com/groksoft/els/stty/hintServer/Daemon.java
@@ -0,0 +1,389 @@
+package com.groksoft.els.stty.hintServer;
+
+import com.groksoft.els.Configuration;
+import com.groksoft.els.Context;
+import com.groksoft.els.Transfer;
+import com.groksoft.els.Utils;
+import com.groksoft.els.repository.HintKeys;
+import com.groksoft.els.repository.Hints;
+import com.groksoft.els.repository.Repository;
+import com.groksoft.els.stty.DaemonBase;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.util.StringTokenizer;
+
+/**
+ * Hint Status Server Daemon service.
+ *
+ * The Daemon service is the command interface used to communicate between
+ * the endpoints.
+ */
+@SuppressWarnings("Duplicates")
+public class Daemon extends DaemonBase
+{
+ protected static Logger logger = LogManager.getLogger("applog");
+
+ private HintKeys.HintKey connectedKey;
+ private Context context;
+ private boolean fault = false;
+ private Hints hints = null;
+ private boolean isTerminal = false;
+
+ /**
+ * Instantiate the Daemon service
+ *
+ * @param config
+ * @param ctxt
+ */
+ public Daemon(Configuration config, Context ctxt, Repository mine, Repository theirs)
+ {
+ super(config, mine, theirs);
+ context = ctxt;
+ } // constructor
+
+ /**
+ * Dump statistics from all available internal sources.
+ */
+ public synchronized String dumpStatistics()
+ {
+ String data = "\r\nConsole currently connected: " + ((connected) ? "true" : "false") + "\r\n";
+ data += " Connected on port: " + port + "\r\n";
+ data += " Connected to: " + address + "\r\n";
+ return data;
+ } // dumpStatistics
+
+ /**
+ * Get the short name of the service.
+ *
+ * @return Short name of this service.
+ */
+ public String getName()
+ {
+ return "Daemon";
+ } // getName
+
+ /**
+ * Get the next available token trimmed
+ *
+ * @param t StringTokenizer
+ * @return Next token or an empty string
+ */
+ private String getNextToken(StringTokenizer t)
+ {
+ String value = "";
+ if (t.hasMoreTokens())
+ {
+ value = t.nextToken();
+ if (value.trim().length() == 0 && t.hasMoreTokens())
+ value = t.nextToken();
+ }
+ return value;
+ }
+
+ /**
+ * Special handshake using hints keys file instead of point-to-point
+ *
+ * @return String name of back-up system
+ */
+ public String handshake()
+ {
+ String system = "";
+ boolean valid = false;
+ try
+ {
+ Utils.writeStream(out, myKey, "HELO");
+
+ String input = Utils.readStream(in, myKey);
+ if (input.equals("DribNit") || input.equals("DribNlt"))
+ {
+ isTerminal = input.equals("DribNit");
+ if (isTerminal) // terminal not allowed for hint status server
+ {
+ Utils.writeStream(out, myKey, "Terminal session not allowed");
+ return system; // empty
+ }
+ Utils.writeStream(out, myKey, myKey);
+
+ input = Utils.readStream(in, myKey);
+ connectedKey = context.hintKeys.findKey(input); // look for matching key in hints keys file
+ if (connectedKey != null)
+ {
+ // send my flavor
+ Utils.writeStream(out, myKey, myRepo.getLibraryData().libraries.flavor);
+
+ system = connectedKey.name;
+ logger.info("Authenticated " + (isTerminal ? "terminal" : "automated") + " session: " + system);
+ valid = true;
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ fault = true;
+ logger.error(e.getMessage());
+ }
+ return system;
+ } // handshake
+
+ /**
+ * Process a connection request to the Daemon service.
+ *
+ * The Daemon service provides an interface for this instance.
+ */
+ public boolean process(Socket aSocket) throws Exception
+ {
+ socket = aSocket;
+ port = aSocket.getPort();
+ address = aSocket.getInetAddress();
+ int attempts = 0;
+ String line;
+ String basePrompt = ": ";
+ String prompt = basePrompt;
+ boolean tout = false;
+
+ // Get ELS hints keys
+ hints = new Hints(cfg, context, context.hintKeys);
+ context.transfer = new Transfer(cfg, context);
+
+ // setup i/o
+ aSocket.setSoTimeout(120000); // time-out so this thread does not hang server
+
+ in = new DataInputStream(aSocket.getInputStream());
+ out = new DataOutputStream(aSocket.getOutputStream());
+
+ connected = true;
+
+ String system = handshake();
+ if (system.length() == 0) // special handshake using hints keys file instead of point-to-point
+ {
+ logger.error("Connection to incoming request failed handshake");
+ }
+ else
+ {
+ if (isTerminal) // terminal not allowed to hint status server in handshake()
+ {
+ response = "Enter 'help' for information\r\n"; // "Enter " checked in ClientStty.checkBannerCommands()
+ }
+ else // is automation
+ {
+ response = "CMD";
+ }
+
+ // prompt for & process interactive commands
+ while (stop == false)
+ {
+ try
+ {
+ // prompt the user for a command
+ if (!tout)
+ {
+ try
+ {
+ Utils.writeStream(out, myKey, response + (isTerminal ? prompt : ""));
+ }
+ catch (Exception e)
+ {
+ logger.info("Client appears to have disconnected");
+ break;
+ }
+ }
+ tout = false;
+ response = "";
+
+ line = readStream(in, myKey); // special readStream() variant for continuous server
+ if (line == null)
+ {
+ // break read loop and let the connection be closed
+ break;
+ }
+
+ if (line.trim().length() < 1)
+ {
+ response = "\r";
+ continue;
+ }
+
+ logger.info("Processing command: " + line + " from: " + system + ", " + Utils.formatAddresses(getSocket()));
+
+ // parse the command
+ StringTokenizer t = new StringTokenizer(line, "\"");
+ if (!t.hasMoreTokens())
+ continue; // ignore if empty
+
+ String theCommand = t.nextToken().trim();
+
+ // -------------- get status --------------------------------
+ if (theCommand.equalsIgnoreCase("get"))
+ {
+ boolean valid = false;
+ if (t.hasMoreTokens())
+ {
+ String itemLib = getNextToken(t);
+ String itemPath = getNextToken(t);
+ String systemName = getNextToken(t);
+ String defaultStatus = getNextToken(t);
+ if (itemLib.length() > 0 && itemPath.length() > 0 && systemName.length() > 0 && defaultStatus.length() > 0)
+ {
+ valid = true;
+ response = context.datastore.getStatus(itemLib, itemPath, systemName, defaultStatus);
+ logger.info(" > get response: " + response);
+ }
+ }
+ if (!valid)
+ response = "false";
+ continue;
+ }
+
+ // -------------- logout ------------------------------------
+ if (theCommand.equalsIgnoreCase("logout"))
+ {
+ if (authorized)
+ {
+ authorized = false;
+ prompt = basePrompt;
+ continue;
+ }
+ else
+ {
+ // break read loop and let the connection be closed
+ break;
+ }
+ }
+
+ // -------------- quit, bye, exit ---------------------------
+ if (theCommand.equalsIgnoreCase("quit") || theCommand.equalsIgnoreCase("bye") || theCommand.equalsIgnoreCase("exit"))
+ {
+ //Utils.writeStream(out, myKey, "End-Execution");
+ stop = true;
+ break; // break the loop
+ }
+
+ // -------------- set status --------------------------------
+ if (theCommand.equalsIgnoreCase("set"))
+ {
+ boolean valid = false;
+ if (t.hasMoreTokens())
+ {
+ String itemLib = getNextToken(t);
+ String itemPath = getNextToken(t);
+ String systemName = getNextToken(t);
+ String status = getNextToken(t);
+ if (itemLib.length() > 0 && itemPath.length() > 0 && systemName.length() > 0 && status.length() > 0)
+ {
+ valid = true;
+ response = context.datastore.setStatus(itemLib, itemPath, systemName, status);
+ logger.info(" > set response: " + response);
+ }
+ }
+ if (!valid)
+ response = "false";
+ continue;
+ }
+
+ response = "\r\nunknown command '" + theCommand + "\r\n";
+
+ } // try
+ catch (Exception e)
+ {
+ fault = true;
+ connected = false;
+ stop = true;
+ logger.error(Utils.getStackTrace(e));
+ try
+ {
+ Utils.writeStream(out, myKey, e.getMessage());
+ }
+ catch (Exception ex)
+ {
+ }
+ break;
+ }
+ }
+ }
+ return stop;
+ } // process
+
+ /**
+ * Continuous Server - Read an encrypted data stream, return decrypted string
+ *
+ * Special variation for operating a continuous server that does not shutdown
+ * if the connection is broken.
+ *
+ * @param in DataInputStream to read, e.g. remote connection
+ * @param key UUID key to decrypt the data stream
+ * @return String read from stream; null if connection is closed
+ */
+ public String readStream(DataInputStream in, String key) throws Exception
+ {
+ byte[] buf = {};
+ String input = "";
+ while (true)
+ {
+ try
+ {
+ int count = in.readInt();
+ int pos = 0;
+ if (count > 0)
+ {
+ buf = new byte[count];
+ int remaining = count;
+ while (true)
+ {
+ int readCount = in.read(buf, pos, remaining);
+ if (readCount < 0)
+ break;
+ pos += readCount;
+ remaining -= readCount;
+ if (pos == count)
+ break;
+ }
+ if (pos != count)
+ {
+ logger.warn("Read counts do not match, expected " + count + ", received " + pos);
+ }
+ }
+ break;
+ }
+ catch (SocketTimeoutException e)
+ {
+ continue;
+ }
+ catch (EOFException e)
+ {
+ input = null;
+ break;
+ }
+ catch (IOException e)
+ {
+ if (e.getMessage().toLowerCase().contains("connection reset"))
+ {
+ logger.info("Connection closed by client");
+ input = null;
+ break;
+ }
+ else // do not throw on disconnect
+ throw e;
+ }
+ }
+ if (buf.length > 0)
+ input = Utils.decrypt(key, buf);
+ return input;
+ }
+
+ /**
+ * Request the Daemon service to stop
+ */
+ public void requestStop()
+ {
+ this.stop = true;
+ logger.debug("Requesting stop for stty session on: " + Utils.formatAddresses(socket));
+ } // requestStop
+
+} // Daemon
diff --git a/src/com/groksoft/els/stty/hintServer/Datastore.java b/src/com/groksoft/els/stty/hintServer/Datastore.java
new file mode 100644
index 00000000..4b4fa640
--- /dev/null
+++ b/src/com/groksoft/els/stty/hintServer/Datastore.java
@@ -0,0 +1,273 @@
+package com.groksoft.els.stty.hintServer;
+
+import com.groksoft.els.Configuration;
+import com.groksoft.els.Context;
+import com.groksoft.els.MungeException;
+import com.groksoft.els.repository.Item;
+import com.groksoft.els.repository.Library;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.MarkerManager;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+
+/**
+ * ELS Hint Status Tracker/Server Datastore class
+ */
+
+public class Datastore
+{
+ private final transient Logger logger = LogManager.getLogger("applog");
+ private final Marker SHORT = MarkerManager.getMarker("SHORT");
+ private final Marker SIMPLE = MarkerManager.getMarker("SIMPLE");
+ private Configuration cfg;
+ private Context context;
+ private String statDirectory;
+ private Library statLibrary;
+
+ /**
+ * Constructor
+ *
+ * @param config Configuration
+ * @param ctx Context
+ */
+ public Datastore(Configuration config, Context ctx)
+ {
+ cfg = config;
+ context = ctx;
+ }
+
+ /**
+ * Add a hint status tracker collection Item and empty file to track a hint's status
+ *
+ * @param itemLib The library of the hint
+ * @param itemPath The path to the new hint file
+ * @param backupName The back-up name for the status
+ * @param defaultStatus The default status for the back-up
+ * @return The new Item for the hint
+ * @throws Exception
+ */
+ private synchronized Item add(String itemLib, String itemPath, String backupName, String defaultStatus) throws Exception
+ {
+ Item item = new Item();
+ item.setLibrary(itemLib);
+ item.setItemPath(itemLib + "--" + itemPath.replaceAll(context.subscriberRepo.getWriteSeparator(), "--"));
+ item.setFullPath(statDirectory + context.statusRepo.getSeparator() + item.getItemPath());
+ item.setSize(42);
+ statLibrary.items.add(item); // add to the in-memory collection
+
+ File stat = new File(item.getFullPath()); // create an empty file
+ stat.createNewFile();
+ logger.info(" + Added hint status file: " + item.getFullPath());
+ return item;
+ }
+
+ /**
+ * Find a back-up name in a hint status file
+ *
+ * @param lines The lines of the hint status file
+ * @param backupName The back-up name to find
+ * @return String line found, or null
+ */
+ private String findBackup(List lines, String backupName)
+ {
+ for (int i = 0; i < lines.size(); ++i)
+ {
+ String line = lines.get(i);
+ if (line.toLowerCase().startsWith(backupName.toLowerCase() + " "))
+ {
+ return line;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Find a hint Item in the hint status tracker collection
+ *
+ * If not found a new Item is added.
+ *
+ * @param itemLib The library of the hint
+ * @param itemPath The path to the new hint file
+ * @param backupName The back-up name for the status
+ * @param defaultStatus The default status for the back-up
+ * @return The Item of the hint
+ * @throws Exception
+ */
+ private Item findItem(String itemLib, String itemPath, String backupName, String defaultStatus) throws Exception
+ {
+ String path = itemLib + "--" + itemPath;
+ Item item = statLibrary.get(path);
+ if (item == null)
+ {
+ item = add(itemLib, itemPath, backupName, defaultStatus);
+ }
+ return item;
+ }
+
+ /**
+ * Command "get" to return the status of a back-up
+ *
+ * @param itemLib The library of the hint
+ * @param itemPath The item path of the hint
+ * @param backupName The back-up name to find
+ * @param defaultStatus The default status if not found
+ * @return The current status of the hint
+ * @throws Exception
+ */
+ public synchronized String getStatus(String itemLib, String itemPath, String backupName, String defaultStatus) throws Exception
+ {
+ String status = "";
+
+ itemPath = itemPath.replaceAll("/", "--").replaceAll("\\\\", "--");
+ Item item = findItem(itemLib, itemPath, backupName, defaultStatus);
+ List lines = Files.readAllLines(Paths.get(item.getFullPath()));
+ String line = findBackup(lines, backupName);
+ if (line == null)
+ {
+ status = defaultStatus;
+ updateDatastore(item, lines, backupName, status);
+ }
+ else
+ {
+ String[] parts = line.split("[\\s]+");
+ if (parts.length == 2)
+ {
+ status = parts[1];
+ }
+ else
+ throw new MungeException("Malformed datastore status line in: " + item.getFullPath());
+ }
+ return status;
+ }
+
+ /**
+ * Initialize the hint tracker.
+ *
+ * Uses the first source from the first library defined in the
+ * hint tracker/server JSON file as the datastore directory for
+ * tracking hint status.
+ *
+ * @throws MungeException
+ */
+ public void initialize() throws MungeException
+ {
+ if (context.statusRepo.getLibraryData() != null &&
+ context.statusRepo.getLibraryData().libraries != null &&
+ context.statusRepo.getLibraryData().libraries.bibliography != null &&
+ context.statusRepo.getLibraryData().libraries.bibliography.length > 0)
+ {
+ statLibrary = context.statusRepo.getLibraryData().libraries.bibliography[0];
+ }
+ else
+ throw new MungeException("Hint Status Tracker/Server repo contains no library for status datastore");
+
+ statDirectory = "";
+ if (statLibrary.sources != null && statLibrary.sources.length > 0)
+ {
+ statDirectory = statLibrary.sources[0];
+ }
+ else
+ throw new MungeException("Hint Status Tracker/Server repo first library contains no sources: " + statLibrary.name);
+
+ File dir = new File(statDirectory);
+ if (dir.exists())
+ {
+ if (!dir.isDirectory())
+ throw new MungeException("Status directory is not a directory: " + statDirectory);
+ logger.info("Using library \'" + statLibrary.name + "\" source directory \"" + statDirectory + "\" for status datastore");
+ }
+ else
+ {
+ logger.info("Creating new library \'" + statLibrary.name + "\" source directory \"" + statDirectory + "\" for status datastore");
+ dir.mkdirs();
+ }
+
+ // scan the status datastore (repository)
+ context.statusRepo.scan(statLibrary.name);
+ }
+
+ /**
+ * Command "set" to change the status of a back-up
+ *
+ * @param itemLib The library of the hint
+ * @param itemPath The item path of the hint
+ * @param backupName The back-up name to find
+ * @param status The status to use
+ * @return The updated status of the hint; should be the same as that passed
+ * @throws Exception
+ */
+ public synchronized String setStatus(String itemLib, String itemPath, String backupName, String status) throws Exception
+ {
+ String path = itemLib + "--" + itemPath;
+ path = path.replaceAll("/", "--").replaceAll("\\\\", "--");
+ Item item = statLibrary.get(path);
+ if (item == null) // if a get() was done first this shouldn't happen
+ {
+ item = add(itemLib, itemPath, backupName, status);
+ }
+
+ List lines = Files.readAllLines(Paths.get(item.getFullPath()));
+ lines = updateDatastore(item, lines, backupName, status);
+ return status;
+ }
+
+ /**
+ * Update the datastore.
+ *
+ * If the hint has been Deleted by all participants then the
+ * hint status tracker/server file is deleted for automatic
+ * hint file maintenance.
+ *
+ * @param item The Item to be updated
+ * @param lines The lines of the item
+ * @param backupName The back-up name to be updated
+ * @param status The status value to be used
+ * @return The updates lines of the hint status tracker/server
+ * @throws Exception
+ */
+ private synchronized List updateDatastore(Item item, List lines, String backupName, String status) throws Exception
+ {
+ int count = 0;
+ int deleted = 0;
+ boolean found = false;
+ for (int i = 0; i < lines.size(); ++i)
+ {
+ String update = "";
+ String line = lines.get(i).trim().toLowerCase();
+ if (line.length() == 0)
+ continue;
+ ++count;
+ if (line.startsWith(backupName.toLowerCase() + " "))
+ {
+ update = backupName + " " + status;
+ lines.set(i, update);
+ line = update.toLowerCase();
+ found = true;
+ }
+ if (line.endsWith(" deleted"))
+ ++deleted;
+ }
+ if (!found)
+ {
+ lines.add(backupName + " " + status);
+ ++count;
+ if (status.trim().equalsIgnoreCase("deleted"))
+ ++deleted;
+ }
+ if (deleted == count)
+ {
+ Files.delete(Paths.get(item.getFullPath()));
+ logger.info(" $ Hint finished by all participants, deleted hint status file: " + item.getFullPath());
+ }
+ else
+ Files.write(Paths.get(item.getFullPath()), lines, StandardOpenOption.CREATE);
+ return lines;
+ }
+
+}
diff --git a/src/com/groksoft/els/stty/publisher/Daemon.java b/src/com/groksoft/els/stty/publisher/Daemon.java
index 368ca4f6..804d8a8a 100755
--- a/src/com/groksoft/els/stty/publisher/Daemon.java
+++ b/src/com/groksoft/els/stty/publisher/Daemon.java
@@ -1,9 +1,7 @@
package com.groksoft.els.stty.publisher;
import com.groksoft.els.*;
-import com.groksoft.els.repository.Item;
-import com.groksoft.els.repository.Library;
-import com.groksoft.els.repository.Repository;
+import com.groksoft.els.repository.*;
import com.groksoft.els.sftp.ClientSftp;
import com.groksoft.els.stty.ClientStty;
import com.groksoft.els.stty.DaemonBase;
@@ -35,6 +33,8 @@ public class Daemon extends DaemonBase
private Context context;
private boolean fault = false;
+ private HintKeys hintKeys = null;
+ private Hints hints = null;
private boolean isTerminal = false;
private Transfer transfer;
@@ -71,17 +71,29 @@ public String getName()
return "Daemon";
} // getName
- public boolean handshake()
+ /**
+ * Perform a point-to-point handshake
+ *
+ * @return String name of back-up system
+ */
+ public String handshake()
{
- boolean valid = false;
+ String system = "";
try
{
Utils.writeStream(out, myKey, "HELO");
String input = Utils.readStream(in, myKey);
- if (input.equals("DribNit") || input.equals("DribNlt"))
+ if (input != null && (input.equals("DribNit") || input.equals("DribNlt")))
{
isTerminal = input.equals("DribNit");
+ if (isTerminal && myRepo.getLibraryData().libraries.terminal_allowed != null &&
+ !myRepo.getLibraryData().libraries.terminal_allowed)
+ {
+ Utils.writeStream(out, myKey, "Terminal session not allowed");
+ logger.warn("Attempt made to login interactively but terminal sessions are not allowed");
+ return system;
+ }
Utils.writeStream(out, myKey, myKey);
input = Utils.readStream(in, myKey);
@@ -90,8 +102,8 @@ public boolean handshake()
// send my flavor
Utils.writeStream(out, myKey, myRepo.getLibraryData().libraries.flavor);
- logger.info("Authenticated " + (isTerminal ? "terminal" : "automated") + " session: " + theirRepo.getLibraryData().libraries.description);
- valid = true;
+ system = theirRepo.getLibraryData().libraries.description;
+ logger.info("Authenticated " + (isTerminal ? "terminal" : "automated") + " session: " + system);
}
}
}
@@ -100,7 +112,7 @@ public boolean handshake()
fault = true;
logger.error(e.getMessage());
}
- return valid;
+ return system;
} // handshake
/**
@@ -108,7 +120,7 @@ public boolean handshake()
*
* The Daemon service provides an interface for this instance.
*/
- public void process(Socket aSocket) throws Exception, IOException
+ public boolean process(Socket aSocket) throws Exception, IOException
{
socket = aSocket;
port = aSocket.getPort();
@@ -123,8 +135,15 @@ public void process(Socket aSocket) throws Exception, IOException
// for get command
long totalSize = 0L;
ArrayList- group = new ArrayList<>();
- transfer = new Transfer(cfg, context); // v3.0.0
- transfer.initialize();
+
+ // Get ELS hints keys if specified
+ if (cfg.getHintKeysFile().length() > 0) // v3.0.0
+ {
+ hintKeys = new HintKeys(cfg, context);
+ hintKeys.read(cfg.getHintKeysFile());
+ hints = new Hints(cfg, context, hintKeys);
+ context.transfer = new Transfer(cfg, context);
+ }
// setup i/o
aSocket.setSoTimeout(120000); // time-out so this thread does not hang server
@@ -134,10 +153,11 @@ public void process(Socket aSocket) throws Exception, IOException
connected = true;
- if (!handshake())
+ String system = handshake();
+ if (system.length() == 0)
{
stop = true; // just hang-up on the connection
- logger.info("Connection to " + theirRepo.getLibraryData().libraries.host + " failed handshake");
+ logger.error("Connection to " + theirRepo.getLibraryData().libraries.host + " failed handshake");
}
else
{
@@ -191,7 +211,7 @@ public void process(Socket aSocket) throws Exception, IOException
continue;
}
- logger.info("Processing command: " + line);
+ logger.info("Processing command: " + line + " from: " + system + ", " + Utils.formatAddresses(getSocket()));
// parse the command
StringTokenizer t = new StringTokenizer(line, "\"");
@@ -517,7 +537,7 @@ public void process(Socket aSocket) throws Exception, IOException
" And:\r\n";
}
- response += " auth [password] = access Authorized commands\r\n" +
+ response += " auth \"password\" = access Authorized commands, enclose password in quote\r\n" +
" collection = get collection data from remote, can take a few moments to scan\r\n" +
" space [location] = free space at location on remote\r\n" +
" targets = get targets file from remote\r\n" +
@@ -541,32 +561,21 @@ public void process(Socket aSocket) throws Exception, IOException
{
Utils.writeStream(out, myKey, e.getMessage());
}
- catch (Exception ex) {}
+ catch (Exception ex)
+ {
+ }
break;
}
- } // while
-
- if (stop)
- {
- // all done, close everything
- if (logger != null)
- {
- logger.info("Close connection on port " + port + " to " + address.getHostAddress());
-
- // mark the process as successful so it may be detected with automation
- if (!fault)
- logger.fatal("Process completed normally");
- else
- logger.fatal("Process failed");
- }
- out.close();
- in.close();
-
- Runtime.getRuntime().exit(0);
}
-
+ return stop;
} // process
+ /**
+ * Collect the remaining tokens into a String
+ *
+ * @param t StringTokenizer
+ * @return String of concatenated tokens
+ */
public String remainingTokens(StringTokenizer t)
{
String result = "";
@@ -583,7 +592,7 @@ public String remainingTokens(StringTokenizer t)
public void requestStop()
{
this.stop = true;
- logger.info("Requesting stop for session on port " + socket.getPort() + " to " + socket.getInetAddress());
+ logger.debug("Requesting stop for stty session on: " + socket.getInetAddress().toString() + ":" + socket.getPort());
} // requestStop
} // Daemon
diff --git a/src/com/groksoft/els/stty/subscriber/Daemon.java b/src/com/groksoft/els/stty/subscriber/Daemon.java
index a562bd52..98637d9f 100755
--- a/src/com/groksoft/els/stty/subscriber/Daemon.java
+++ b/src/com/groksoft/els/stty/subscriber/Daemon.java
@@ -69,6 +69,13 @@ public String getName()
return "Daemon";
} // getName
+
+ /**
+ * Get the next available token trimmed
+ *
+ * @param t StringTokenizer
+ * @return Next token or an empty string
+ */
private String getNextToken(StringTokenizer t)
{
String value = "";
@@ -81,17 +88,29 @@ private String getNextToken(StringTokenizer t)
return value;
}
- public boolean handshake()
+ /**
+ * Perform a point-to-point handshake
+ *
+ * @return String name of back-up system
+ */
+ public String handshake()
{
- boolean valid = false;
+ String system = "";
try
{
Utils.writeStream(out, myKey, "HELO");
String input = Utils.readStream(in, myKey);
- if (input.equals("DribNit") || input.equals("DribNlt"))
+ if (input != null && (input.equals("DribNit") || input.equals("DribNlt")))
{
isTerminal = input.equals("DribNit");
+ if (isTerminal && myRepo.getLibraryData().libraries.terminal_allowed != null &&
+ !myRepo.getLibraryData().libraries.terminal_allowed)
+ {
+ Utils.writeStream(out, myKey, "Terminal session not allowed");
+ logger.warn("Attempt made to login interactively but terminal sessions are not allowed");
+ return system;
+ }
Utils.writeStream(out, myKey, myKey);
input = Utils.readStream(in, myKey);
@@ -100,8 +119,8 @@ public boolean handshake()
// send my flavor
Utils.writeStream(out, myKey, myRepo.getLibraryData().libraries.flavor);
- logger.info("Authenticated " + (isTerminal ? "terminal" : "automated") + " session: " + theirRepo.getLibraryData().libraries.description);
- valid = true;
+ system = theirRepo.getLibraryData().libraries.description;
+ logger.info("Authenticated " + (isTerminal ? "terminal" : "automated") + " session: " + system);
}
}
}
@@ -110,7 +129,7 @@ public boolean handshake()
fault = true;
logger.error(e.getMessage());
}
- return valid;
+ return system;
} // handshake
/**
@@ -118,7 +137,7 @@ public boolean handshake()
*
* The Daemon service provides an interface for this instance.
*/
- public void process(Socket aSocket) throws Exception
+ public boolean process(Socket aSocket) throws Exception
{
socket = aSocket;
port = aSocket.getPort();
@@ -132,7 +151,7 @@ public void process(Socket aSocket) throws Exception
// Get ELS hints keys if specified
if (cfg.getHintKeysFile().length() > 0) // v3.0.0
{
- hintKeys = new HintKeys(context);
+ hintKeys = new HintKeys(cfg, context);
hintKeys.read(cfg.getHintKeysFile());
hints = new Hints(cfg, context, hintKeys);
context.transfer = new Transfer(cfg, context);
@@ -146,10 +165,11 @@ public void process(Socket aSocket) throws Exception
connected = true;
- if (!handshake())
+ String system = handshake();
+ if (system.length() == 0)
{
stop = true; // just hang-up on the connection
- logger.info("Connection to " + theirRepo.getLibraryData().libraries.host + " failed handshake");
+ logger.error("Connection to " + theirRepo.getLibraryData().libraries.host + " failed handshake");
}
else
{
@@ -203,7 +223,7 @@ public void process(Socket aSocket) throws Exception
continue;
}
- logger.info("Processing command: " + line);
+ logger.info("Processing command: " + line + " from: " + system + ", " + Utils.formatAddresses(getSocket()));
// parse the command
StringTokenizer t = new StringTokenizer(line, "\"");
@@ -309,7 +329,7 @@ public void process(Socket aSocket) throws Exception
if (libName.length() > 0 && itemPath.length() > 0 && toPath.length() > 0)
{
valid = true;
- boolean sense = hints.hintExecute(libName, itemPath, toPath);
+ boolean sense = hints.hintRun(libName, itemPath, toPath);
response = (isTerminal ? "ok" + (sense ? ", executed" : "") + "\r\n" : Boolean.toString(sense));
}
}
@@ -418,7 +438,7 @@ public void process(Socket aSocket) throws Exception
" And:\r\n";
}
- response += " auth [password] = access Authorized commands\r\n" +
+ response += " auth \"password\" = access Authorized commands, enclose password in quote\r\n" +
" collection = get collection data from remote, can take a few moments to scan\r\n" +
" space [location] = free space at location on remote\r\n" +
" targets = get targets file from remote\r\n" +
@@ -448,27 +468,8 @@ public void process(Socket aSocket) throws Exception
}
break;
}
- } // while
-
- if (stop)
- {
- // all done, close everything
- if (logger != null)
- {
- logger.info("Close connection on port " + port + " to " + address.getHostAddress());
-
- // mark the process as successful so it may be detected with automation
- if (!fault)
- logger.fatal("Process completed normally");
- else
- logger.fatal("Process failed");
- }
- out.close();
- in.close();
-
- Runtime.getRuntime().exit(0);
}
-
+ return stop;
} // process
/**
@@ -477,7 +478,7 @@ public void process(Socket aSocket) throws Exception
public void requestStop()
{
this.stop = true;
- logger.info("Requesting stop for session on port " + socket.getPort() + " to " + socket.getInetAddress());
+ logger.debug("Requesting stop for stty session on " + socket.getInetAddress().toString() + ":" + socket.getPort());
} // requestStop
} // Daemon