feat: add --archive option

closes #2

README.md

@@ -20,18 +20,24 @@
 
 ## Options
 
-| Option                  | Type   | Required | Description                                                                                |
-| ----------------------- | ------ | -------- | ------------------------------------------------------------------------------------------ |
-| --url                   | String | true     | URL to podcast RSS feed.                                                                   |
-| --out-dir               | String | false    | Specify output directory for episodes and metadata. Defaults to current working directory. |
-| --include-meta          |        | false    | Write out podcast metadata to JSON.                                                        |
-| --include-episode-meta  |        | false    | Write out individual episode metadata to JSON.                                             |
-| --ignore-episode-images |        | false    | Ignore downloading found images from --include-episode-meta.                               |
-| --offset                | Number | false    | Offset starting download position. Default is 0.                                           |
-| --limit                 | Number | false    | Max number of episodes to download. Downloads all by default.                              |
-| --reverse               |        | false    | Reverse download direction and start at last RSS item.                                     |
-| --info                  |        | false    | Print retrieved podcast info instead of downloading.                                       |
-| --list                  |        | false    | Print episode list instead of downloading.                                                 |
-| --prompt                |        | false    | Use CLI prompt to select options. Ignores other provided options.                          |
-| --version               |        | false    | Output the version number.                                                                 |
-| --help                  |        | false    | Output usage information.                                                                  |
+| Option                  | Type   | Required | Description                                                                                         |
+| ----------------------- | ------ | -------- | --------------------------------------------------------------------------------------------------- |
+| --url                   | String | true     | URL to podcast RSS feed.                                                                            |
+| --out-dir               | String | false    | Specify output directory for episodes and metadata. Defaults to current working directory.          |
+| --archive               | String | false    | Download or write out items not listed in archive file. Generates archive file at path if not found |
+| --include-meta          |        | false    | Write out podcast metadata to JSON.                                                                 |
+| --include-episode-meta  |        | false    | Write out individual episode metadata to JSON.                                                      |
+| --ignore-episode-images |        | false    | Ignore downloading found images from --include-episode-meta.                                        |
+| --offset                | Number | false    | Offset starting download position. Default is 0.                                                    |
+| --limit                 | Number | false    | Max number of episodes to download. Downloads all by default.                                       |
+| --reverse               |        | false    | Reverse download direction and start at last RSS item.                                              |
+| --info                  |        | false    | Print retrieved podcast info instead of downloading.                                                |
+| --list                  |        | false    | Print episode list instead of downloading.                                                          |
+| --prompt                |        | false    | Use CLI prompt to select options. Ignores other provided options.                                   |
+| --version               |        | false    | Output the version number.                                                                          |
+| --help                  |        | false    | Output usage information.                                                                           |
+
+## Archive
+
+- If passed the `--archive <path>` option, `podcast-dl` will generate/use a JSON archive at the provided path.
+- Before downloading an episode or writing out metadata, it'll check if the item was saved previously and abort the save if found.

bin/bin.js

@@ -1,12 +1,14 @@
 #!/usr/bin/env node
 
-let path = require("path");
+let _path = require("path");
+let _url = require("url");
 let commander = require("commander");
 
 let { version } = require("../package.json");
 let { startPrompt } = require("./prompt");
 let {
   download,
+  getArchiveKey,
   getEpisodeAudioUrl,
   getEpisodeFilename,
   getFeed,
@@ -19,12 +21,22 @@ let {
   writeFeedMeta,
   writeItemMeta,
 } = require("./util");
-let { createParseNumber, logError, logErrorAndExit } = require("./validate");
+let {
+  createParseNumber,
+  logError,
+  logErrorAndExit,
+  parseArchivePath,
+} = require("./validate");
 
 commander
   .version(version)
   .option("--url <string>", "url to podcast rss feed")
   .option("--out-dir <path>", "specify output directory", "./")
+  .option(
+    "--archive <path>",
+    "download or write only items not listed in archive file",
+    parseArchivePath
+  )
   .option("--include-meta", "write out podcast metadata to json")
   .option(
     "--include-episode-meta",
@@ -52,6 +64,7 @@ commander
   .parse(process.argv);
 
 let {
+  archive,
   url,
   outDir,
   includeMeta,
@@ -75,7 +88,9 @@ let main = async () => {
     logErrorAndExit("No URL provided");
   }
 
-  let basePath = path.resolve(process.cwd(), outDir);
+  let { hostname, pathname } = _url.parse(url);
+  let archiveUrl = `${hostname}${pathname}`;
+  let basePath = _path.resolve(process.cwd(), outDir);
   let feed = await getFeed(url);
 
   if (info) {
@@ -99,14 +114,14 @@ let main = async () => {
 
     if (podcastImageUrl) {
       let podcastImageFileExt = getUrlExt(podcastImageUrl);
-      let outputImagePath = path.resolve(
-        basePath,
-        `image${podcastImageFileExt}`
-      );
+      let podcastImageName = `image${podcastImageFileExt}`;
+      let outputImagePath = _path.resolve(basePath, podcastImageName);
 
       try {
         console.log("Saving podcast image");
         await download({
+          archive,
+          key: getArchiveKey({ prefix: archiveUrl, name: podcastImageName }),
           outputPath: outputImagePath,
           url: podcastImageUrl,
         });
@@ -117,10 +132,16 @@ let main = async () => {
       logError("Unable to find podcast image");
     }
 
-    let outputMetaPath = path.resolve(basePath, `meta.json`);
+    let outputMetaName = "meta.json";
+    let outputMetaPath = _path.resolve(basePath, outputMetaName);
 
     console.log("Saving podcast metadata");
-    writeFeedMeta({ outputPath: outputMetaPath, feed });
+    writeFeedMeta({
+      archive,
+      feed,
+      key: getArchiveKey({ prefix: archiveUrl, name: outputMetaName }),
+      outputPath: outputMetaPath,
+    });
   }
 
   if (!feed.items || feed.items.length === 0) {
@@ -159,13 +180,13 @@ let main = async () => {
 
     let baseSafeFilename = getEpisodeFilename(item);
     let audioFileExt = getUrlExt(episodeAudioUrl);
-    let outputPodcastPath = path.resolve(
-      basePath,
-      `${baseSafeFilename}${audioFileExt}`
-    );
+    let episodeName = `${baseSafeFilename}${audioFileExt}`;
+    let outputPodcastPath = _path.resolve(basePath, episodeName);
 
     try {
       await download({
+        archive,
+        key: getArchiveKey({ prefix: archiveUrl, name: episodeName }),
         outputPath: outputPodcastPath,
         url: episodeAudioUrl,
       });
@@ -179,15 +200,17 @@ let main = async () => {
 
         if (episodeImageUrl) {
           let episodeImageFileExt = getUrlExt(episodeImageUrl);
-          let outputImagePath = path.resolve(
-            basePath,
-            `${baseSafeFilename}${episodeImageFileExt}`
-          );
+          let episodeImageName = `${baseSafeFilename}${episodeImageFileExt}`;
+          let outputImagePath = _path.resolve(basePath, episodeName);
 
           console.log("Saving episode image");
-
           try {
             await download({
+              archive,
+              key: getArchiveKey({
+                prefix: archiveUrl,
+                name: episodeImageName,
+              }),
               outputPath: outputImagePath,
               url: episodeImageUrl,
             });
@@ -199,13 +222,16 @@ let main = async () => {
         }
       }
 
-      let outputEpisodeMetaPath = path.resolve(
-        basePath,
-        `${baseSafeFilename}.meta.json`
-      );
+      let episodeMetaName = `${baseSafeFilename}.meta.json`;
+      let outputEpisodeMetaPath = _path.resolve(basePath, episodeMetaName);
 
       console.log("Saving episode metadata");
-      writeItemMeta({ outputPath: outputEpisodeMetaPath, item });
+      writeItemMeta({
+        archive,
+        item,
+        key: getArchiveKey({ prefix: archiveUrl, name: episodeMetaName }),
+        outputPath: outputEpisodeMetaPath,
+      });
     }
 
     console.log("");

bin/util.js

@@ -41,7 +41,42 @@ let logItemInfo = (item) => {
   console.log(`Publish Date: ${pubDate}`);
 };
 
-let writeFeedMeta = ({ outputPath, feed }) => {
+let getArchiveKey = ({ prefix, name }) => {
+  return `${prefix}-${name}`;
+};
+
+let writeToArchive = ({ key, archive }) => {
+  let archiveResult = [];
+  let archivePath = path.resolve(process.cwd(), archive);
+
+  if (fs.existsSync(archivePath)) {
+    archiveResult = JSON.parse(fs.readFileSync(archivePath));
+  }
+
+  if (!archiveResult.includes(key)) {
+    archiveResult.push(key);
+  }
+
+  fs.writeFileSync(archivePath, JSON.stringify(archiveResult, null, 4));
+};
+
+let getIsInArchive = ({ key, archive }) => {
+  let archivePath = path.resolve(process.cwd(), archive);
+
+  if (!fs.existsSync(archivePath)) {
+    return false;
+  }
+
+  let archiveResult = JSON.parse(fs.readFileSync(archivePath));
+  return archiveResult.includes(key);
+};
+
+let writeFeedMeta = ({ outputPath, feed, key, archive }) => {
+  if (key && archive && getIsInArchive({ key, archive })) {
+    console.log("Feed metadata exists in archive. Skipping write");
+    return;
+  }
+
   let title = feed.title || null;
   let description = feed.description || null;
   let link = feed.link || null;
@@ -63,12 +98,21 @@ let writeFeedMeta = ({ outputPath, feed }) => {
         4
       )
     );
+
+    if (key && archive && !getIsInArchive({ key, archive })) {
+      writeToArchive({ key, archive });
+    }
   } catch (error) {
-    logError("Unable to save meta file for episode", error);
+    logError("Unable to save metadata file for episode", error);
   }
 };
 
-let writeItemMeta = ({ outputPath, item }) => {
+let writeItemMeta = ({ outputPath, item, key, archive }) => {
+  if (key && archive && getIsInArchive({ key, archive })) {
+    console.log("Episode metadata exists in archive. Skipping write");
+    return;
+  }
+
   let title = item.title || null;
   let descriptionText = item.contentSnippet || null;
   let pubDate = item.pubDate || null;
@@ -88,29 +132,33 @@ let writeItemMeta = ({ outputPath, item }) => {
         4
       )
     );
+
+    if (key && archive && !getIsInArchive({ key, archive })) {
+      writeToArchive({ key, archive });
+    }
   } catch (error) {
     logError("Unable to save meta file for episode", error);
   }
 };
 
 let getUrlExt = (url) => {
-  let pathname = _url.parse(url).pathname;
+  let { pathname } = _url.parse(url);
   let ext = path.extname(pathname);
   return ext;
 };
 
 let VALID_AUDIO_TYPES = [".mp3", ".aac", ".m4a", ".wav", ".ogg", ".flac"];
-let isAudioUrl = (url) => {
+let getIsAudioUrl = (url) => {
   let ext = getUrlExt(url);
   return VALID_AUDIO_TYPES.includes(ext);
 };
 
 let getEpisodeAudioUrl = ({ enclosure, link }) => {
-  if (link && isAudioUrl(link)) {
+  if (link && getIsAudioUrl(link)) {
     return link;
   }
 
-  if (enclosure && isAudioUrl(enclosure.url)) {
+  if (enclosure && getIsAudioUrl(enclosure.url)) {
     return enclosure.url;
   }
 
@@ -168,9 +216,15 @@ let printProgress = ({ percent, total }) => {
 
 let endPrintProgress = () => {
   process.stdout.write("\n");
+  currentProgressLine = "";
 };
 
-let download = async ({ url, outputPath }) => {
+let download = async ({ url, outputPath, key, archive }) => {
+  if (key && archive && getIsInArchive({ key, archive })) {
+    console.log("Download exists in archive. Skipping");
+    return;
+  }
+
   await pipeline(
     got
       .stream(url)
@@ -182,6 +236,10 @@ let download = async ({ url, outputPath }) => {
       }),
     fs.createWriteStream(outputPath)
   );
+
+  if (key && archive && !getIsInArchive({ key, archive })) {
+    writeToArchive({ key, archive });
+  }
 };
 
 let getLoopControls = ({ limit, offset, length, reverse }) => {
@@ -229,6 +287,7 @@ let getFeed = async (url) => {
 
 module.exports = {
   download,
+  getArchiveKey,
   getEpisodeAudioUrl,
   getEpisodeFilename,
   getFeed,

bin/validate.js

@@ -35,8 +35,17 @@ const createParseNumber = ({ min, name, required = true }) => {
   };
 };
 
+const parseArchivePath = (value) => {
+  if (!value.length) {
+    logErrorAndExit("Must provide --archive path");
+  }
+
+  return value;
+};
+
 module.exports = {
   createParseNumber,
   logError,
   logErrorAndExit,
+  parseArchivePath,
 };