From 1bd1c5b2a8c8abd03908c8190dccfc1ad9b08931 Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 10:35:01 -0500 Subject: [PATCH 01/15] Checkpoint --- CHANGELOG.md | 2 +- README.md | 2 +- data/default/rules/github.yml | 4 + src/bin/noseyparker/args.rs | 148 +++++++++++------- src/bin/noseyparker/cmd_github.rs | 88 +---------- src/bin/noseyparker/cmd_scan.rs | 108 ++++++++++--- src/datastore.rs | 11 ++ src/git_binary.rs | 119 ++++++++++++++ src/github/client.rs | 30 +++- src/github/client_builder.rs | 30 ++++ src/github/error.rs | 3 + src/github/mod.rs | 47 ++++++ src/github/repo_enumerator.rs | 73 +++++++++ src/input_enumerator.rs | 2 +- src/lib.rs | 5 +- tests/cli.rs | 4 +- .../cli__noseyparker_help_github-2.snap | 5 +- .../cli__noseyparker_help_scan-2.snap | 59 +++++-- 18 files changed, 549 insertions(+), 191 deletions(-) create mode 100644 src/git_binary.rs create mode 100644 src/github/repo_enumerator.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ef26cbf3f..76831afdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Additions - Nosey Parker now has rudimentary support for enumerating repositories from GitHub users and organizations. The new `github repos list` command uses the GitHub REST API to enumerate repositories belonging to one or more users or organizations. - An optional GitHub Personal Access Token can be provided via the `GITHUB_TOKEN` environment variable. + An optional GitHub Personal Access Token can be provided via the `NP_GITHUB_TOKEN` environment variable. - Nosey Parker now has an optional `rule_profiling` crate feature that causes performance-related statistics to be collected and reported when scanning. This feature imposes some performance cost and is only useful to rule authors, and so is disabled by default. diff --git a/README.md b/README.md index 935f774b8..28922d3f6 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ https://github.com/octocat/octocat.github.io.git https://github.com/octocat/test-repo1.git ``` -An optional GitHub Personal Access Token can be provided via the `GITHUB_TOKEN` environment variable. +An optional GitHub Personal Access Token can be provided via the `NP_GITHUB_TOKEN` environment variable. Providing an access token gives a higher API rate limit and may make additional repositories accessible to you. Additional output formats are supported, including JSON and JSON lines, via the `--format=FORMAT` option. diff --git a/data/default/rules/github.yml b/data/default/rules/github.yml index b6c7e8dd3..1d7bd0685 100644 --- a/data/default/rules/github.yml +++ b/data/default/rules/github.yml @@ -30,6 +30,7 @@ rules: - name: GitHub App Token + # Note: `ghu_` prefix is for user-to-server tokens; `ghs_` is for server-to-server tokens pattern: '\b((?:ghu|ghs)_[a-zA-Z0-9]{36})\b' references: @@ -39,6 +40,9 @@ rules: examples: - ' "token": "ghu_16C7e42F292c69C2E7C10c838347Ae178B4a",' + - | + Example usage: + git clone http://ghs_RguXIkihJjwHAP6eXEYxaPNvywurTr5IOAbg@github.com/username/repo.git - name: GitHub Refresh Token diff --git a/src/bin/noseyparker/args.rs b/src/bin/noseyparker/args.rs index 055b62bf3..4dd87dcc4 100644 --- a/src/bin/noseyparker/args.rs +++ b/src/bin/noseyparker/args.rs @@ -1,5 +1,5 @@ use anyhow::{Context, Result}; -use clap::{ArgAction, Args, Parser, Subcommand, ValueEnum, crate_version, crate_description}; +use clap::{crate_description, crate_version, ArgAction, Args, Parser, Subcommand, ValueEnum}; use std::path::PathBuf; @@ -52,17 +52,32 @@ impl CommandLineArgs { pub enum Command { /// Scan content for secrets /// - /// This command uses regex-based rules to identify hardcoded secrets and other potentially - /// sensitive information in textual content (or in inputs that can have textual content - /// extracted from them). + /// This command uses regex-based rules to identify hardcoded secrets and other potentially sensitive information in textual content (or in inputs that can have textual content extracted from them). /// - /// The inputs can be either files or directories. - /// Files are scanned directly; directories are recursively enumerated and scanned. - /// Any Git repositories encountered will have their entire history scanned. + /// The findings from scanning are recorded into a datastore. The recorded findings can later + /// be reported in several formats using the `summarize` and `report` commands. /// - /// The findings from scanning are recorded into a datastore. - /// The recorded findings can later be reported in several formats using the `summarize` and - /// `report` commands. + /// Several types of inputs can be specified: + /// + /// - Positional input arguments can be either files or directories. + /// Files are scanned directly; directories are recursively enumerated and scanned. + /// Any directories encountered that are Git repositories will have their entire history scanned. + /// + /// - A Git repository URL can be specified with the `--git-repo=URL` argument. + /// This will cause Nosey Parker to clone that repository to its datastore and scan its history. + /// + /// - A GitHub user can be specified with the `--github-user=NAME` argument. + /// This will cause Nosey Parker to enumerate accessible repositories belonging to that user, clone them to its datastore, and scan their entire history. + /// + /// - A GitHub organization can be specified with the `--github-org=NAME` argument. + /// This will cause Nosey Parker to enumerate accessible repositories belonging to that organization, clone them to its datastore, and scan their entire history. + /// + /// The `git` binary on the PATH is used to clone any required Git repositories. + /// It is careful invoked to avoid using any system-wide or user-specific configuration. + /// + /// By default, when cloning repositories from GitHub or enumerating GitHub users or organizations, unauthenticated access is used. + /// An optional personal access token can be specified using the `NP_GITHUB_TOKEN` environment variable. + /// Using a personal access token gives higher rate limits and may make additional content accessible. #[command(display_order = 1)] Scan(ScanArgs), @@ -76,10 +91,9 @@ pub enum Command { /// Interact with GitHub /// - /// An optional personal access token can be specified using the `GITHUB_TOKEN` environment - /// variable. - /// Using a personal access token gives higher rate limits and may make additional content - /// accessible. + /// By default, unauthenticated access is used. + /// An optional personal access token can be specified using the `NP_GITHUB_TOKEN` environment variable. + /// Using a personal access token gives higher rate limits and may make additional content accessible. #[command(display_order = 4, name = "github")] GitHub(GitHubArgs), @@ -96,7 +110,7 @@ pub enum Command { // global options // ----------------------------------------------------------------------------- #[derive(Args, Debug)] -#[command(next_help_heading="Global Options")] +#[command(next_help_heading = "Global Options")] pub struct GlobalArgs { /// Enable verbose output /// @@ -108,8 +122,7 @@ pub struct GlobalArgs { /// /// When this is "auto", colors are enabled when stdout is a tty. /// - /// If the `NO_COLOR` environment variable is set, it takes precedence and is equivalent to - /// `--color=never`. + /// If the `NO_COLOR` environment variable is set, it takes precedence and is equivalent to `--color=never`. #[arg(global=true, long, default_value_t=Mode::Auto, value_name="MODE")] pub color: Mode, @@ -187,7 +200,7 @@ pub struct GitHubReposListArgs { pub output_args: OutputArgs, } -#[derive(Args, Debug)] +#[derive(Args, Debug, Clone)] pub struct GitHubRepoSpecifiers { /// Select repositories belonging to the specified user #[arg(long)] @@ -204,7 +217,6 @@ impl GitHubRepoSpecifiers { } } - // ----------------------------------------------------------------------------- // `rules` command // ----------------------------------------------------------------------------- @@ -218,14 +230,13 @@ pub struct RulesArgs { pub enum RulesCommand { /// Check rules for problems /// - /// If errors are detected, or if warnings are detected and `--warnings-as-errors` is passed, - /// the program will exit with a nonzero exit code. + /// If errors are detected or if warnings are detected and `--warnings-as-errors` is specified, the program will exit with a nonzero exit code. Check(RulesCheckArgs), } #[derive(Args, Debug)] pub struct RulesCheckArgs { - #[arg(long, short='W')] + #[arg(long, short = 'W')] /// Treat warnings as errors pub warnings_as_errors: bool, @@ -251,29 +262,29 @@ pub enum DatastoreCommand { #[derive(Args, Debug)] pub struct DatastoreInitArgs { - #[arg(long, short, value_name="PATH", env("NP_DATASTORE"))] + #[arg(long, short, value_name = "PATH", env("NP_DATASTORE"))] /// Initialize the datastore at specified path pub datastore: PathBuf, } - fn get_parallelism() -> usize { match std::thread::available_parallelism() { - Err(_e) => { 1 } - Ok(v) => { v.into() } + Err(_e) => 1, + Ok(v) => v.into(), } } // ----------------------------------------------------------------------------- // `scan` command // ----------------------------------------------------------------------------- +/// Arguments for the `scan` command #[derive(Args, Debug)] pub struct ScanArgs { /// Use the specified datastore path /// /// The datastore will be created if it does not exist. - #[arg(long, short, value_name="PATH", env("NP_DATASTORE"))] - // FIXME: choose a default value for this + #[arg(long, short, value_name = "PATH", env("NP_DATASTORE"))] + // FIXME: choose a good default value for this? pub datastore: PathBuf, /// The number of parallel scanning jobs @@ -283,35 +294,58 @@ pub struct ScanArgs { /// Path of custom rules to use /// /// The paths can be either files or directories. - /// Directories are recursively walked and all found rule files will be loaded. + /// Directories are recursively walked and all discovered rule files will be loaded. /// /// This option can be repeated. - #[arg(long, short, value_name="PATH")] + #[arg(long, short, value_name = "PATH")] pub rules: Vec, - /// Paths of inputs to scan - /// - /// Inputs can be files, directories, or Git repositories. - #[arg(num_args(1..), required(true), value_name="INPUT")] - pub inputs: Vec, + #[command(flatten)] + pub input_args: ScanInputArgs, #[command(flatten)] - pub discovery_args: DiscoveryArgs, + pub discovery_args: ScanDiscoveryArgs, } -// ----------------------------------------------------------------------------- -// enumeration options -// ----------------------------------------------------------------------------- #[derive(Args, Debug)] -#[command(next_help_heading="Content Discovery Options")] -pub struct DiscoveryArgs { +#[command(next_help_heading = "Input Specifier Options")] +pub struct ScanInputArgs { + /// Path to a file, directory, or local Git repository to scan + #[arg(value_name="INPUT", required_unless_present_any(["github_user", "github_organization", "git_repo"]), display_order=1)] + pub path_inputs: Vec, + + /// URL of a Git repository to clone and scan + #[arg(long, value_name = "URL", display_order = 10)] + pub git_repo: Vec, + + /// Name of a GitHub user to enumerate and scan + #[arg(long, value_name = "NAME", display_order = 20)] + pub github_user: Vec, + + /// Name of a GitHub organization to enumerate and scan + #[arg( + long, + visible_alias = "github-org", + value_name = "NAME", + display_order = 20 + )] + pub github_organization: Vec, +} + +/// This struct represents options to control content discovery. +#[derive(Args, Debug)] +#[command(next_help_heading = "Content Discovery Options")] +pub struct ScanDiscoveryArgs { /// Do not scan files larger than the specified size /// - /// The value is parsed as a floating point literal, and hence can be non-integral. + /// The value is parsed as a floating point literal, and hence fractional values can be supplied. /// A negative value means "no limit". - /// Note that scanning requires reading the entire contents of each file into memory, - /// so using an excessively large limit may be problematic. - #[arg(long("max-file-size"), default_value_t=100.0, value_name="MEGABYTES")] + /// Note that scanning requires reading the entire contents of each file into memory, so using an excessively large limit may be problematic. + #[arg( + long("max-file-size"), + default_value_t = 100.0, + value_name = "MEGABYTES" + )] pub max_file_size_mb: f64, /// Path of a custom ignore rules file to use @@ -319,18 +353,16 @@ pub struct DiscoveryArgs { /// The ignore file should contain gitignore-style rules. /// /// This option can be repeated. - #[arg(long, short, value_name="FILE")] + #[arg(long, short, value_name = "FILE")] pub ignore: Vec, - /* /// Do not scan files that appear to be binary #[arg(long)] pub skip_binary_files: bool, */ - } -impl DiscoveryArgs { +impl ScanDiscoveryArgs { pub fn max_file_size_bytes(&self) -> Option { if self.max_file_size_mb < 0.0 { None @@ -346,7 +378,7 @@ impl DiscoveryArgs { #[derive(Args, Debug)] pub struct SummarizeArgs { /// Use the specified datastore path - #[arg(long, short, value_name="PATH", env("NP_DATASTORE"))] + #[arg(long, short, value_name = "PATH", env("NP_DATASTORE"))] pub datastore: PathBuf, #[command(flatten)] @@ -359,7 +391,7 @@ pub struct SummarizeArgs { #[derive(Args, Debug)] pub struct ReportArgs { /// Use the specified datastore path - #[arg(long, short, value_name="PATH", env("NP_DATASTORE"))] + #[arg(long, short, value_name = "PATH", env("NP_DATASTORE"))] pub datastore: PathBuf, #[command(flatten)] @@ -370,12 +402,12 @@ pub struct ReportArgs { // output options // ----------------------------------------------------------------------------- #[derive(Args, Debug)] -#[command(next_help_heading="Output Options")] +#[command(next_help_heading = "Output Options")] pub struct OutputArgs { /// Write output to the specified path /// /// If this argument is not provided, stdout will be used. - #[arg(long, short, value_name="PATH")] + #[arg(long, short, value_name = "PATH")] pub output: Option, /// Write output in the specified format @@ -391,9 +423,7 @@ impl OutputArgs { use std::io::BufWriter; match &self.output { - None => { - Ok(Box::new(BufWriter::new(std::io::stdout()))) - } + None => Ok(Box::new(BufWriter::new(std::io::stdout()))), Some(p) => { let f = File::create(p)?; Ok(Box::new(BufWriter::new(f))) @@ -430,7 +460,6 @@ impl std::fmt::Display for OutputFormat { } } - // ----------------------------------------------------------------------------- // report writer // ----------------------------------------------------------------------------- @@ -440,7 +469,8 @@ pub trait Reportable { fn jsonl_format(&self, writer: W) -> Result<()>; fn report(&self, output_args: &OutputArgs) -> Result<()> { - let writer = output_args.get_writer() + let writer = output_args + .get_writer() .context("Failed to open output destination for writing")?; let result = match &output_args.format { @@ -454,7 +484,7 @@ pub trait Reportable { // Ignore SIGPIPE errors, like those that can come from piping to `head` Some(e) if e.kind() == std::io::ErrorKind::BrokenPipe => Ok(()), _ => Err(e)?, - } + }, } } } diff --git a/src/bin/noseyparker/cmd_github.rs b/src/bin/noseyparker/cmd_github.rs index 1804a02a6..adfe2983f 100644 --- a/src/bin/noseyparker/cmd_github.rs +++ b/src/bin/noseyparker/cmd_github.rs @@ -1,7 +1,6 @@ -use anyhow::{bail, Context, Result}; -use tracing::{debug, warn}; +use anyhow::{bail, Result}; -use crate::args::{GlobalArgs, GitHubArgs, GitHubReposListArgs, Reportable}; +use crate::args::{GitHubArgs, GitHubReposListArgs, GlobalArgs, Reportable}; use noseyparker::github; pub fn run(global_args: &GlobalArgs, args: &GitHubArgs) -> Result<()> { @@ -11,90 +10,17 @@ pub fn run(global_args: &GlobalArgs, args: &GitHubArgs) -> Result<()> { } } -/// The name of the environment variable to look for a personal access token in. -/// -/// NOTE: this variable needs to match the top-level help documentation in args.rs -const GITHUB_TOKEN_ENV_VAR: &str = "GITHUB_TOKEN"; - fn list_repos(_global_args: &GlobalArgs, args: &GitHubReposListArgs) -> Result<()> { if args.repo_specifiers.is_empty() { bail!("No repositories specified"); } - - let client = { - let mut builder = github::ClientBuilder::new(); - match std::env::var(GITHUB_TOKEN_ENV_VAR) { - Err(std::env::VarError::NotPresent) => { - debug!("No GitHub access token provided; using unauthenticated API access."); - } - Err(std::env::VarError::NotUnicode(_s)) => { - bail!("Value of {} environment variable is ill-formed", GITHUB_TOKEN_ENV_VAR); - } - Ok(val) => { - debug!("Using GitHub personal access token from {GITHUB_TOKEN_ENV_VAR} environment variable"); - builder = builder - .auth(github::Auth::PersonalAccessToken(secrecy::SecretString::from(val))); - } - } - builder - .build() - .context("Failed to initialize GitHub client")? - }; - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .context("Failed to initialize async runtime")?; - - let result = runtime.block_on(async { - let mut repo_urls: Vec = Vec::new(); - - // Get rate limit first thing. - // If there are connectivity issues, this is likely to reveal them quickly. - // - // This also makes it a little bit simpler to test this code, as the very first request - // made by `github repos list` will always be the same, regardless of which repo specifiers - // are given. - let rate_limit = client.get_rate_limit().await?; - debug!("GitHub rate limits: {:?}", rate_limit.rate); - - for username in &args.repo_specifiers.user { - let mut repo_page = Some(client.get_user_repos(username).await?); - while let Some(page) = repo_page { - repo_urls.extend(page.items.iter().map(|r| &r.clone_url).cloned()); - repo_page = client.next_page(page).await?; - } - } - - for orgname in &args.repo_specifiers.organization { - let mut repo_page = Some(client.get_org_repos(orgname).await?); - while let Some(page) = repo_page { - repo_urls.extend(page.items.iter().map(|r| &r.clone_url).cloned()); - repo_page = client.next_page(page).await?; - } - } - - repo_urls.sort(); - repo_urls.dedup(); - - Ok::, noseyparker::github::Error>(repo_urls) - }); - - match result { - Ok(repo_urls) => { - RepoReporter(repo_urls).report(&args.output_args)?; - } - Err(noseyparker::github::Error::RateLimited { wait, .. }) => { - warn!("Rate limit exceeded: Would need to wait for {:?} before retrying", wait); - result?; - } - Err(err) => bail!(err), - } - - Ok(()) + let repo_urls = github::enumerate_repo_urls(&github::RepoSpecifiers { + user: args.repo_specifiers.user.clone(), + organization: args.repo_specifiers.organization.clone(), + })?; + RepoReporter(repo_urls).report(&args.output_args) } - struct RepoReporter(Vec); impl Reportable for RepoReporter { diff --git a/src/bin/noseyparker/cmd_scan.rs b/src/bin/noseyparker/cmd_scan.rs index 044575da2..32a388271 100644 --- a/src/bin/noseyparker/cmd_scan.rs +++ b/src/bin/noseyparker/cmd_scan.rs @@ -13,7 +13,9 @@ use noseyparker::blob::Blob; use noseyparker::blob_id_set::BlobIdSet; use noseyparker::datastore::Datastore; use noseyparker::defaults::DEFAULT_IGNORE_RULES; -use noseyparker::input_enumerator::{FileResult, FilesystemEnumerator, open_git_repo}; +use noseyparker::github; +use noseyparker::git_binary::Git; +use noseyparker::input_enumerator::{open_git_repo, FileResult, FilesystemEnumerator}; use noseyparker::location; use noseyparker::match_type::Match; use noseyparker::matcher::{BlobMatch, Matcher}; @@ -29,7 +31,7 @@ use noseyparker::rules_database::RulesDatabase; pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> { let _span = debug_span!("scan").entered(); - debug!("Args: {:?}", args); + debug!("Args: {args:#?}"); let color_enabled = global_args.use_color(); let progress_enabled = global_args.use_progress(); @@ -48,18 +50,7 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> // Open datastore // --------------------------------------------------------------------------------------------- let mut datastore = Datastore::create_or_open(&args.datastore)?; - - // --------------------------------------------------------------------------------------------- - // Get temporary directory - // --------------------------------------------------------------------------------------------- let tmpdir = datastore.tmpdir(); - std::fs::create_dir_all(&tmpdir).with_context(|| { - format!( - "Failed to create temporary directory {} for datastore at {}", - tmpdir.display(), - datastore.root_dir().display() - ) - })?; // --------------------------------------------------------------------------------------------- // Load rules @@ -74,14 +65,71 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> RulesDatabase::from_rules(rules).context("Failed to compile rules")? }; + // --------------------------------------------------------------------------------------------- + // Enumerate any mentioned GitHub repositories; gather list of all repos to clone or update + // --------------------------------------------------------------------------------------------- + let repo_urls = { + let repo_specifiers = github::RepoSpecifiers { + user: args.input_args.github_user.clone(), + organization: args.input_args.github_organization.clone(), + }; + let mut repo_urls = args.input_args.git_repo.clone(); + if !repo_specifiers.is_empty() { + repo_urls.extend(github::enumerate_repo_urls(&repo_specifiers)?); + } + repo_urls.sort(); + repo_urls.dedup(); + repo_urls + }; + + // --------------------------------------------------------------------------------------------- + // Clone or update all mentioned Git URLs + // --------------------------------------------------------------------------------------------- + debug!("{} Git URLs to clone or update", repo_urls.len()); + for repo_url in &repo_urls { + debug!("Need to clone or update {repo_url}") + } + + let mut input_roots = args.input_args.path_inputs.clone(); + + if !repo_urls.is_empty() { + let clones_dir = tmpdir.join("clones"); + let git = Git::new(); + + for repo_url in repo_urls { + let output_dir = clones_dir.join(&repo_url); + + // First, try to update an existing clone, and if that fails, do a fresh clone + if output_dir.is_dir() { + match git.update_mirrored_clone(&repo_url, &output_dir) { + Ok(()) => { + input_roots.push(output_dir); + continue; + }, + Err(e) => { + debug!("Failed to update clone of {repo_url} at {}: {e}", output_dir.display()); + std::fs::remove_dir_all(&output_dir)?; + } + } + } + + if let Err(e) = git.create_fresh_mirrored_clone(&repo_url, &output_dir) { + error!("Failed to clone {repo_url} to {}: {e}", output_dir.display()); + continue; + } + input_roots.push(output_dir); + } + } + // --------------------------------------------------------------------------------------------- // Enumerate initial filesystem inputs // --------------------------------------------------------------------------------------------- + debug_assert!(!input_roots.is_empty()); let inputs = { let mut progress = Progress::new_bytes_spinner("Enumerating inputs...", progress_enabled); - let input_enumerator = { - let mut ie = FilesystemEnumerator::new(&args.inputs)?; + let input_enumerator = || -> Result { + let mut ie = FilesystemEnumerator::new(&input_roots)?; ie.threads(args.num_jobs); ie.max_filesize(args.discovery_args.max_file_size_bytes()); @@ -104,10 +152,13 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> })?; } - ie - }; + Ok(ie) + }() + .context("Failed to initialize filesystem enumerator")?; - let inputs = input_enumerator.run(&progress)?; + let inputs = input_enumerator + .run(&progress) + .context("Failed to enumerate filesystem inputs")?; let total_bytes_found: u64 = { let blob_bytes: u64 = inputs.git_repos.iter().map(|r| r.total_blob_bytes()).sum(); let file_bytes: u64 = inputs.files.iter().map(|e| e.num_bytes).sum(); @@ -138,7 +189,7 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> Matcher::new(&rules_db, &seen_blobs, Some(&matcher_stats)) }; - let snippet_context_bytes: usize = 128; // FIXME:parameterize this and expose to CLI + let snippet_context_bytes: usize = 128; // FIXME: parameterize this and expose to CLI // a function to convert BlobMatch into regular Match let convert_blob_matches = @@ -237,11 +288,17 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> let repository = match open_git_repo(&git_repo_result.path) { Ok(Some(repository)) => repository.into_sync(), Ok(None) => { - error!("Failed to re-open previously-found repository at {}", git_repo_result.path.display()); + error!( + "Failed to re-open previously-found repository at {}", + git_repo_result.path.display() + ); return; } Err(err) => { - error!("Failed to re-open previously-found repository at {}: {err}", git_repo_result.path.display()); + error!( + "Failed to re-open previously-found repository at {}: {err}", + git_repo_result.path.display() + ); return; } }; @@ -271,7 +328,7 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> ); return; } - // TODO: get rid of this extra copy + // FIXME: get rid of this extra copy Ok(blob) => Blob::new(*blob_id, blob.data.to_owned()), }; let provenance = Provenance::GitRepo { @@ -343,7 +400,12 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> .get_rule(entry.rule_id) .expect("rule index should be valid") .name; - println!("{:>50} {:>10} {:>10.4}s", rule_name, entry.raw_match_count, entry.stage2_duration.as_secs_f64()); + println!( + "{:>50} {:>10} {:>10.4}s", + rule_name, + entry.raw_match_count, + entry.stage2_duration.as_secs_f64() + ); } } diff --git a/src/datastore.rs b/src/datastore.rs index 2b2254a23..f2d8b51e3 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -43,6 +43,16 @@ impl Datastore { }; ds.migrate() .with_context(|| format!("Failed to migrate database at {}", db_path.display()))?; + + let tmpdir = ds.tmpdir(); + std::fs::create_dir_all(&tmpdir).with_context(|| { + format!( + "Failed to create temporary directory {} for datastore at {}", + tmpdir.display(), + ds.root_dir().display() + ) + })?; + Ok(ds) } @@ -61,6 +71,7 @@ impl Datastore { Self::open(root_dir) } + /// Get the path to this datastore's temporary directory. pub fn tmpdir(&self) -> PathBuf { self.root_dir.join("scratch") } diff --git a/src/git_binary.rs b/src/git_binary.rs new file mode 100644 index 000000000..551383168 --- /dev/null +++ b/src/git_binary.rs @@ -0,0 +1,119 @@ +use std::path::Path; +use std::process::{Command, ExitStatus, Stdio}; +use tracing::{debug, debug_span}; + +#[derive(Debug)] +pub enum GitError +{ + IOError(std::io::Error), + GitError { + stdout: Vec, + stderr: Vec, + status: ExitStatus, + }, +} + +impl From for GitError { + fn from(err: std::io::Error) -> GitError { + GitError::IOError(err) + } +} + +impl std::fmt::Display for GitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + GitError::IOError(e) => write!(f, "{e}"), + GitError::GitError { + stdout: _, + stderr: _, + status, + } => write!(f, "git execution failed: {}", status), + } + } +} + +impl std::error::Error for GitError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + GitError::IOError(e) => Some(e), + GitError::GitError {..} => None, + } + } +} + +pub struct Git { + credentials: Vec, +} + +impl Git { + pub fn new() -> Self { + let credentials: Vec = // if std::env::var("NP_GITHUB_TOKEN").is_ok() { + [ + "-c", + r#"credential.helper="#, + "-c", + r#"credential.helper=!_ghcreds() { echo username="$NP_GITHUB_TOKEN"; echo password=; }; _ghcreds"#, + ].iter().map(|s| s.to_string()).collect() + // } else { + // vec![] + // }; + ; + + Self { credentials } + } + + fn git(&self) -> Command { + let mut cmd = Command::new("git"); + cmd.env("GIT_CONFIG_GLOBAL", "/dev/null"); + cmd.env("GIT_CONFIG_NOSYSTEM", "1"); + cmd.env("GIT_CONFIG_SYSTEM", "/dev/null"); + cmd.args(&self.credentials); + cmd.stdin(Stdio::null()); + cmd + } + + pub fn update_mirrored_clone(&self, repo_url: &str, output_dir: &Path) -> Result<(), GitError> { + let _span = debug_span!("git_update", "{repo_url} {}", output_dir.display()).entered(); + debug!("Attempting to update clone of {repo_url} at {}", output_dir.display()); + + let mut cmd = self.git(); + cmd.arg("--git-dir") + .arg(output_dir) + .arg("remote") + .arg("update") + .arg("--prune"); + + debug!("{cmd:#?}"); + let output = cmd.output()?; + if !output.status.success() { + return Err(GitError::GitError { + stdout: output.stdout, + stderr: output.stderr, + status: output.status, + }); + } + Ok(()) + } + + pub fn create_fresh_mirrored_clone(&self, repo_url: &str, output_dir: &Path) -> Result<(), GitError> { + let _span = debug_span!("git_clone", "{repo_url} {}", output_dir.display()).entered(); + debug!("Attempting to create fresh clone of {} at {}", repo_url, output_dir.display()); + + let mut cmd = self.git(); + cmd.arg("clone") + .arg("--mirror") + .arg(&repo_url) + .arg(output_dir); + + debug!("{cmd:#?}"); + let output = cmd.output()?; + if !output.status.success() { + return Err(GitError::GitError { + stdout: output.stdout, + stderr: output.stderr, + status: output.status, + }); + } + Ok(()) + } +} diff --git a/src/github/client.rs b/src/github/client.rs index 6045d7426..8834034b0 100644 --- a/src/github/client.rs +++ b/src/github/client.rs @@ -59,12 +59,32 @@ impl Client { where T: serde::de::DeserializeOwned, { - if let Some(next) = page.links.next { - let response = self.get_url(next).await?; - Ok(Some(Page::from_response(response).await?)) - } else { - Ok(None) + self.next_page_inner(page.links.next).await + } + + async fn next_page_inner(&self, next: Option) -> Result>> + where + T: serde::de::DeserializeOwned, + { + match next { + Some(next) => { + let response = self.get_url(next).await?; + Ok(Some(Page::from_response(response).await?)) + } + None => Ok(None), + } + } + + pub async fn get_all(&self, page: Page) -> Result> + where T: serde::de::DeserializeOwned + { + let mut results = Vec::new(); + let mut next_page = Some(page); + while let Some(page) = next_page { + results.extend(page.items.into_iter()); + next_page = self.next_page_inner(page.links.next).await?; } + Ok(results) } } diff --git a/src/github/client_builder.rs b/src/github/client_builder.rs index 3d4d7265c..bedf36f37 100644 --- a/src/github/client_builder.rs +++ b/src/github/client_builder.rs @@ -1,7 +1,9 @@ use reqwest::{IntoUrl, Url}; +use tracing::debug; use super::{Auth, Client, Error, Result}; + // ------------------------------------------------------------------------------------------------- // ClientBuilder // ------------------------------------------------------------------------------------------------- @@ -11,8 +13,10 @@ pub struct ClientBuilder { } impl ClientBuilder { + /// The user agent string sent when accessing the GitHub REST API const USER_AGENT: &str = "noseyparker"; + /// Create a new `ClientBuilder` that uses unauthenticated access to https://api.github.com. pub fn new() -> Self { ClientBuilder { base_url: Url::parse("https://api.github.com").expect("default base URL should parse"), @@ -20,16 +24,41 @@ impl ClientBuilder { } } + /// Use the specified base URL. pub fn base_url(mut self, url: T) -> Result { self.base_url = url.into_url().map_err(Error::ReqwestError)?; Ok(self) } + /// Use the given authentication mechanism. pub fn auth(mut self, auth: Auth) -> Self { self.auth = auth; self } + /// Load an optional personal access token token from the `NP_GITHUB_TOKEN` environment variable. + /// If that variable is not set, unauthenticated access is used. + pub fn personal_access_token_from_env(self) -> Result { + self.personal_access_token_from_env_var("NP_GITHUB_TOKEN") + } + + fn personal_access_token_from_env_var(mut self, env_var_name: &str) -> Result { + match std::env::var(env_var_name) { + Err(std::env::VarError::NotPresent) => { + debug!("No GitHub access token provided; using unauthenticated API access."); + } + Err(std::env::VarError::NotUnicode(_s)) => { + return Err(Error::InvalidTokenEnvVar(env_var_name.to_string())); + } + Ok(val) => { + debug!("Using GitHub personal access token from {env_var_name} environment variable"); + self.auth = Auth::PersonalAccessToken(secrecy::SecretString::from(val)); + } + } + Ok(self) + } + + /// Build a `Client` from this `ClientBuilder`. pub fn build(self) -> Result { let inner = reqwest::ClientBuilder::new() .user_agent(Self::USER_AGENT) @@ -44,6 +73,7 @@ impl ClientBuilder { } impl Default for ClientBuilder { + /// Equivalent to `ClientBuilder::new()`. fn default() -> Self { Self::new() } diff --git a/src/github/error.rs b/src/github/error.rs index 430b3a3dc..2cc115e17 100644 --- a/src/github/error.rs +++ b/src/github/error.rs @@ -16,6 +16,7 @@ pub enum Error { UrlParseError(url::ParseError), UrlSlashError(String), ReqwestError(reqwest::Error), + InvalidTokenEnvVar(String), } impl std::fmt::Display for Error { @@ -25,6 +26,7 @@ impl std::fmt::Display for Error { Error::UrlParseError(e) => write!(f, "error parsing URL: {e}"), Error::UrlSlashError(p) => write!(f, "error building URL: component {p:?} contains a slash"), Error::ReqwestError(e) => write!(f, "error making request: {e}"), + Error::InvalidTokenEnvVar(v) => write!(f, "error loading token: ill-formed value of {v} environment variable"), } } } @@ -36,6 +38,7 @@ impl std::error::Error for Error { Error::UrlParseError(e) => Some(e), Error::UrlSlashError(_) => None, Error::ReqwestError(e) => Some(e), + Error::InvalidTokenEnvVar(_) => None, } } } diff --git a/src/github/mod.rs b/src/github/mod.rs index 9c642a7fb..2d129304f 100644 --- a/src/github/mod.rs +++ b/src/github/mod.rs @@ -3,10 +3,57 @@ mod client; mod client_builder; mod error; mod models; +mod repo_enumerator; mod result; pub use auth::Auth; pub use client::Client; pub use client_builder::ClientBuilder; pub use error::Error; +pub use repo_enumerator::{RepoEnumerator, RepoSpecifiers}; pub use result::Result; + +/// List accessible repository URLs matching the given specifiers. +/// +/// This is a high-level wrapper for enumerating GitHub repositories that handles the details of +/// creating an async runtime and a GitHub REST API client. +pub fn enumerate_repo_urls(repo_specifiers: &RepoSpecifiers) -> anyhow::Result> { + use anyhow::{bail, Context}; + use tracing::{debug, warn}; + + let client = ClientBuilder::new() + .personal_access_token_from_env() + .context("Failed to load access token from environment")? + .build() + .context("Failed to initialize GitHub client")?; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("Failed to initialize async runtime")?; + + let result = runtime.block_on(async { + // Get rate limit first thing. + // If there are connectivity issues, this is likely to reveal them quickly. + // + // This also makes it a little bit simpler to test this code, as the very first request + // made by `github repos list` will always be the same, regardless of which repo specifiers + // are given. + let rate_limit = client.get_rate_limit().await?; + debug!("GitHub rate limits: {:?}", rate_limit.rate); + + let repo_enumerator = RepoEnumerator::new(&client); + let repo_urls = repo_enumerator.enumerate_repo_urls(repo_specifiers).await?; + Ok::, Error>(repo_urls) + }); + + match result { + Ok(repo_urls) => Ok(repo_urls), + Err(err) => { + if let Error::RateLimited { wait, .. } = err { + warn!("Rate limit exceeded: Would need to wait for {wait:?} before retrying"); + } + bail!(err); + } + } +} diff --git a/src/github/repo_enumerator.rs b/src/github/repo_enumerator.rs new file mode 100644 index 000000000..0567122b2 --- /dev/null +++ b/src/github/repo_enumerator.rs @@ -0,0 +1,73 @@ +use super::models::Repository; +use super::{Client, Result}; + +/// A `RepoEnumerator` provides higher-level functionality on top of the GitHub REST API to list +/// repositories belonging to specific users or organizations. +pub struct RepoEnumerator<'c> { + client: &'c Client, +} + +impl<'c> RepoEnumerator<'c> { + pub fn new(client: &'c Client) -> Self { + Self { client } + } + + /// Enumerate the accessible repositories that belong to the given user. + pub async fn enumerate_user_repos(&self, username: &str) -> Result> { + let repo_page = self.client.get_user_repos(username).await?; + self.client.get_all(repo_page).await + } + + /// Enumerate the accessible repositories that belong to the given organization. + pub async fn enumerate_org_repos(&self, orgname: &str) -> Result> { + let repo_page = self.client.get_org_repos(orgname).await?; + self.client.get_all(repo_page).await + } + + /// Enumerate the repository clone URLs found from the according to the given `RepoSpecifiers`, + /// collecting the union of specified repository URLs. + /// + /// The resulting URLs are sorted and deduplicated. + pub async fn enumerate_repo_urls( + &self, + repo_specifiers: &RepoSpecifiers, + ) -> Result> { + let mut repo_urls = Vec::new(); + + for username in &repo_specifiers.user { + repo_urls.extend( + self.enumerate_user_repos(username) + .await? + .into_iter() + .map(|r| r.clone_url), + ); + } + + for orgname in &repo_specifiers.organization { + repo_urls.extend( + self.enumerate_org_repos(orgname) + .await? + .into_iter() + .map(|r| r.clone_url), + ); + } + + repo_urls.sort(); + repo_urls.dedup(); + + Ok(repo_urls) + } +} + +/// Specifies a set of GitHub usernames and/or organization names. +#[derive(Debug)] +pub struct RepoSpecifiers { + pub user: Vec, + pub organization: Vec, +} + +impl RepoSpecifiers { + pub fn is_empty(&self) -> bool { + self.user.is_empty() && self.organization.is_empty() + } +} diff --git a/src/input_enumerator.rs b/src/input_enumerator.rs index a65fa4fad..a110d2f05 100644 --- a/src/input_enumerator.rs +++ b/src/input_enumerator.rs @@ -188,7 +188,7 @@ impl FilesystemEnumerator { pub fn new>(inputs: &[T]) -> Result { if inputs.is_empty() { - bail!("No inputs provided"); + bail!("No root inputs provided"); } let mut builder = WalkBuilder::new(&inputs[0]); diff --git a/src/lib.rs b/src/lib.rs index 2167fcb70..c1dd69d68 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod blob_id_set; pub mod bstring_escape; pub mod datastore; pub mod defaults; +pub mod git_binary; pub mod github; pub mod input_enumerator; pub mod location; @@ -12,9 +13,9 @@ pub mod matcher; pub mod matcher_stats; pub mod progress; pub mod provenance; -pub mod rules; -pub mod rules_database; #[cfg(feature = "rule_profiling")] pub mod rule_profiling; +pub mod rules; +pub mod rules_database; pub mod snippet; pub mod utils; diff --git a/tests/cli.rs b/tests/cli.rs index 1e00f5297..90c6056fa 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -229,7 +229,7 @@ fn test_noseyparker_github_repos_list_noargs() { fn test_noseyparker_github_repos_list_org_badtoken() { let cmd = noseyparker() .args(&["github", "repos", "list", "--org", "praetorian-inc"]) - .env("GITHUB_TOKEN", "hahabogus") + .env("NP_GITHUB_TOKEN", "hahabogus") .assert() .failure(); assert_cmd_snapshot!(cmd); @@ -239,7 +239,7 @@ fn test_noseyparker_github_repos_list_org_badtoken() { fn test_noseyparker_github_repos_list_user_badtoken() { let cmd = noseyparker() .args(&["github", "repos", "list", "--user", "octocat"]) - .env("GITHUB_TOKEN", "hahabogus") + .env("NP_GITHUB_TOKEN", "hahabogus") .assert() .failure(); assert_cmd_snapshot!(cmd); diff --git a/tests/snapshots/cli__noseyparker_help_github-2.snap b/tests/snapshots/cli__noseyparker_help_github-2.snap index fdd4e8b37..b28332e55 100644 --- a/tests/snapshots/cli__noseyparker_help_github-2.snap +++ b/tests/snapshots/cli__noseyparker_help_github-2.snap @@ -4,8 +4,9 @@ expression: stdout --- Interact with GitHub -An optional personal access token can be specified using the `GITHUB_TOKEN` environment variable. -Using a personal access token gives higher rate limits and may make additional content accessible. +By default, unauthenticated access is used. An optional personal access token can be specified using +the `NP_GITHUB_TOKEN` environment variable. Using a personal access token gives higher rate limits +and may make additional content accessible. Usage: noseyparker github [OPTIONS] diff --git a/tests/snapshots/cli__noseyparker_help_scan-2.snap b/tests/snapshots/cli__noseyparker_help_scan-2.snap index f209dd50c..17dbee429 100644 --- a/tests/snapshots/cli__noseyparker_help_scan-2.snap +++ b/tests/snapshots/cli__noseyparker_help_scan-2.snap @@ -7,20 +7,35 @@ Scan content for secrets This command uses regex-based rules to identify hardcoded secrets and other potentially sensitive information in textual content (or in inputs that can have textual content extracted from them). -The inputs can be either files or directories. Files are scanned directly; directories are -recursively enumerated and scanned. Any Git repositories encountered will have their entire history -scanned. - The findings from scanning are recorded into a datastore. The recorded findings can later be reported in several formats using the `summarize` and `report` commands. -Usage: noseyparker scan [OPTIONS] --datastore ... +Several types of inputs can be specified: -Arguments: - ... - Paths of inputs to scan - - Inputs can be files, directories, or Git repositories. +- Positional input arguments can be either files or directories. Files are scanned directly; +directories are recursively enumerated and scanned. Any directories encountered that are Git +repositories will have their entire history scanned. + +- A Git repository URL can be specified with the `--git-repo=URL` argument. This will cause Nosey +Parker to clone that repository to its datastore and scan its history. + +- A GitHub user can be specified with the `--github-user=NAME` argument. This will cause Nosey +Parker to enumerate accessible repositories belonging to that user, clone them to its datastore, and +scan their entire history. + +- A GitHub organization can be specified with the `--github-org=NAME` argument. This will cause +Nosey Parker to enumerate accessible repositories belonging to that organization, clone them to its +datastore, and scan their entire history. + +The `git` binary on the PATH is used to clone any required Git repositories. It is careful invoked +to avoid using any system-wide or user-specific configuration. + +By default, when cloning repositories from GitHub or enumerating GitHub users or organizations, +unauthenticated access is used. An optional personal access token can be specified using the +`NP_GITHUB_TOKEN` environment variable. Using a personal access token gives higher rate limits and +may make additional content accessible. + +Usage: noseyparker scan [OPTIONS] --datastore [INPUT]... Options: -d, --datastore @@ -39,20 +54,36 @@ Options: Path of custom rules to use The paths can be either files or directories. Directories are recursively walked and all - found rule files will be loaded. + discovered rule files will be loaded. This option can be repeated. -h, --help Print help (see a summary with '-h') +Input Specifier Options: + [INPUT]... + Path to a file, directory, or local Git repository to scan + + --git-repo + URL of a Git repository to clone and scan + + --github-organization + Name of a GitHub organization to enumerate and scan + + [aliases: github-org] + + --github-user + Name of a GitHub user to enumerate and scan + Content Discovery Options: --max-file-size Do not scan files larger than the specified size - The value is parsed as a floating point literal, and hence can be non-integral. A negative - value means "no limit". Note that scanning requires reading the entire contents of each - file into memory, so using an excessively large limit may be problematic. + The value is parsed as a floating point literal, and hence fractional values can be + supplied. A negative value means "no limit". Note that scanning requires reading the + entire contents of each file into memory, so using an excessively large limit may be + problematic. [default: 100] From 60ac5918b92735e8dc0ffd756889a2204f90c40e Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 11:25:14 -0500 Subject: [PATCH 02/15] Fix Dockerfile: use release profile instead of dev --- Dockerfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f5966e8f..a28f09b4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG VECTORSCAN_SHA=71fae7ee8d63e1513a6df762cdb5d5f02a9120a2422cf1f31d57747c2b8d3 ################################################################################ # Base stage ################################################################################ -FROM rust:$RUST_VER AS base +FROM rust:$RUST_VER AS base_builder ARG VECTORSCAN_VER ARG VECTORSCAN_SHA @@ -37,7 +37,13 @@ RUN apt-get update &&\ ################################################################################ # Build Rust dependencies, caching stage ################################################################################ -FROM base AS dependencies +# This stage exists so that dependencies of Nosey Parker can be preserved in +# the Docker cache. +# +# Building dependencies only is not naturally supported out-of-the box with +# Cargo, and so requires some machinations. + +FROM base_builder AS dependencies_builder WORKDIR "/noseyparker" @@ -57,12 +63,12 @@ RUN mkdir -p ./src/bin/noseyparker &&\ # Stub main required for compile echo "fn main() {}" > ./src/bin/noseyparker/main.rs &&\ # Run the build - cargo build --release + cargo build --release --profile release --locked ################################################################################ # Build application ################################################################################ -FROM dependencies AS build +FROM dependencies_builder AS app_builder WORKDIR "/noseyparker" @@ -76,14 +82,14 @@ RUN touch \ ./src/lib.rs \ ./src/bin/noseyparker/main.rs -RUN cargo build --release +RUN cargo install --root /usr/local --profile release --locked --path . ################################################################################ # Build a smaller image just for running the `noseyparker` binary ################################################################################ FROM debian:11-slim -COPY --from=build /noseyparker/target/release/noseyparker /usr/bin/noseyparker +COPY --from=app_builder /usr/local/bin/noseyparker /usr/local/bin/noseyparker # Tip when running: use a volume mount: `-v "$PWD:/scan"` to make for handling of paths on the command line WORKDIR "/scan" From 563d6c83e064862243949035ac3a3cad30752fe3 Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 11:25:24 -0500 Subject: [PATCH 03/15] Fix documentation link --- src/github/client_builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github/client_builder.rs b/src/github/client_builder.rs index bedf36f37..bae0a9e48 100644 --- a/src/github/client_builder.rs +++ b/src/github/client_builder.rs @@ -16,7 +16,7 @@ impl ClientBuilder { /// The user agent string sent when accessing the GitHub REST API const USER_AGENT: &str = "noseyparker"; - /// Create a new `ClientBuilder` that uses unauthenticated access to https://api.github.com. + /// Create a new `ClientBuilder` that uses unauthenticated access to . pub fn new() -> Self { ClientBuilder { base_url: Url::parse("https://api.github.com").expect("default base URL should parse"), From d197dc00ed32cc2c7b3307624d970ea09b43fc55 Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 12:17:06 -0500 Subject: [PATCH 04/15] Use the `native-tls-vendored` feature of `reqwest` to avoid a runtime dependency on openssl on linux --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e4113f6fd..4b382097c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ pretty_assertions = "1.3" prettytable-rs = "0.10" rayon = "1.5" regex = "1.7" -reqwest = { version = "0.11", features = ["json"] } +reqwest = { version = "0.11", features = ["json", "native-tls-vendored"] } rlimit = "0.9.0" rusqlite = { version = "0.28", features = ["bundled", "backup"] } secrecy = "0.8.0" From 3d7882b1bb602b225238203995059beecc984af9 Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 14:36:33 -0500 Subject: [PATCH 05/15] Fix a couple clippy nits --- src/git_binary.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/git_binary.rs b/src/git_binary.rs index 551383168..80b84c2d3 100644 --- a/src/git_binary.rs +++ b/src/git_binary.rs @@ -102,7 +102,7 @@ impl Git { let mut cmd = self.git(); cmd.arg("clone") .arg("--mirror") - .arg(&repo_url) + .arg(repo_url) .arg(output_dir); debug!("{cmd:#?}"); @@ -117,3 +117,10 @@ impl Git { Ok(()) } } + +impl Default for Git { + /// Equivalent to `Git::new()` + fn default() -> Self { + Self::new() + } +} From b34e48b11619274bce91c206c42bdb9137b951ab Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 15:53:00 -0500 Subject: [PATCH 06/15] Break up integration tests; add test cases; refine GitHub integration tests --- .github/workflows/ci.yml | 5 + src/{github/mod.rs => github.rs} | 0 src/github/{models/mod.rs => models.rs} | 0 tests/cli.rs | 419 ------------------ tests/common/mod.rs | 170 +++++++ tests/github.rs | 83 ++++ tests/help.rs | 92 ++++ tests/scan.rs | 117 +++++ tests/scan_workflow.rs | 30 ++ ...eyparker_github_repos_list_badtoken-3.snap | 9 - .../cli__noseyparker_help_report-3.snap | 5 - .../cli__noseyparker_help_report.snap | 5 - .../cli__noseyparker_help_rules-3.snap | 5 - .../cli__noseyparker_help_rules.snap | 5 - .../cli__noseyparker_help_scan-3.snap | 5 - .../snapshots/cli__noseyparker_help_scan.snap | 5 - .../cli__noseyparker_help_summarize-3.snap | 5 - .../cli__noseyparker_help_summarize.snap | 5 - .../snapshots/cli__noseyparker_no_args-2.snap | 5 - tests/snapshots/cli__noseyparker_no_args.snap | 5 - .../cli__noseyparker_scan_emptydir-3.snap | 5 - .../cli__noseyparker_scan_emptydir.snap | 5 - .../cli__noseyparker_scan_secrets1-3.snap | 5 - .../cli__noseyparker_scan_secrets1-4.snap | 5 - .../cli__noseyparker_scan_secrets1-6.snap | 5 - .../cli__noseyparker_scan_secrets1.snap | 5 - tests/snapshots/cli__status.snap | 5 - tests/snapshots/cli__stderr.snap | 5 - tests/snapshots/cli__stdout.snap | 56 --- ...> github__github_repos_list_noargs-2.snap} | 2 +- ...> github__github_repos_list_noargs-3.snap} | 2 +- ... => github__github_repos_list_noargs.snap} | 2 +- ...ub__github_repos_list_org_badtoken-2.snap} | 2 +- ...ub__github_repos_list_org_badtoken-3.snap} | 2 +- ...thub__github_repos_list_org_badtoken.snap} | 2 +- ...b__github_repos_list_user_badtoken-2.snap} | 2 +- ...b__github_repos_list_user_badtoken-3.snap} | 2 +- ...hub__github_repos_list_user_badtoken.snap} | 2 +- ...eyparker_help-2.snap => help__help-2.snap} | 2 +- ...eyparker_help-3.snap => help__help-3.snap} | 2 +- ..._noseyparker_help.snap => help__help.snap} | 2 +- ...ore-2.snap => help__help_datastore-2.snap} | 2 +- ...ore-3.snap => help__help_datastore-3.snap} | 2 +- ...tastore.snap => help__help_datastore.snap} | 2 +- ...github-2.snap => help__help_github-2.snap} | 2 +- ..._repos-3.snap => help__help_github-3.snap} | 2 +- ...elp_github.snap => help__help_github.snap} | 2 +- ...-2.snap => help__help_github_repos-2.snap} | 2 +- ...-3.snap => help__help_github_repos-3.snap} | 2 +- ...epos.snap => help__help_github_repos.snap} | 2 +- .../help__help_github_repos_short-2.snap | 22 + .../help__help_github_repos_short-3.snap | 5 + .../help__help_github_repos_short.snap | 5 + .../snapshots/help__help_github_short-2.snap | 22 + .../snapshots/help__help_github_short-3.snap | 5 + tests/snapshots/help__help_github_short.snap | 5 + ...report-2.snap => help__help_report-2.snap} | 2 +- tests/snapshots/help__help_report-3.snap | 5 + tests/snapshots/help__help_report.snap | 5 + .../snapshots/help__help_report_short-2.snap | 24 + .../snapshots/help__help_report_short-3.snap | 5 + tests/snapshots/help__help_report_short.snap | 5 + ...p_rules-2.snap => help__help_rules-2.snap} | 2 +- tests/snapshots/help__help_rules-3.snap | 5 + tests/snapshots/help__help_rules.snap | 5 + ...elp_scan-2.snap => help__help_scan-2.snap} | 2 +- tests/snapshots/help__help_scan-3.snap | 5 + tests/snapshots/help__help_scan.snap | 5 + tests/snapshots/help__help_scan_short-2.snap | 32 ++ tests/snapshots/help__help_scan_short-3.snap | 5 + tests/snapshots/help__help_scan_short.snap | 5 + tests/snapshots/help__help_short-2.snap | 28 ++ tests/snapshots/help__help_short-3.snap | 5 + tests/snapshots/help__help_short.snap | 5 + ...ize-2.snap => help__help_summarize-2.snap} | 2 +- tests/snapshots/help__help_summarize-3.snap | 5 + tests/snapshots/help__help_summarize.snap | 5 + .../help__help_summarize_short-2.snap | 24 + .../help__help_summarize_short-3.snap | 5 + .../snapshots/help__help_summarize_short.snap | 5 + ...ist_noargs-2.snap => help__no_args-2.snap} | 2 +- ...er_no_args-3.snap => help__no_args-3.snap} | 2 +- ...t_org_badtoken.snap => help__no_args.snap} | 2 +- ...ap => scan_workflow__scan_secrets1-2.snap} | 2 +- .../scan_workflow__scan_secrets1-3.snap | 5 + .../scan_workflow__scan_secrets1-4.snap | 5 + ...ap => scan_workflow__scan_secrets1-5.snap} | 2 +- .../scan_workflow__scan_secrets1-6.snap | 5 + ...ap => scan_workflow__scan_secrets1-7.snap} | 2 +- .../scan_workflow__scan_secrets1.snap | 5 + 90 files changed, 800 insertions(+), 605 deletions(-) rename src/{github/mod.rs => github.rs} (100%) rename src/github/{models/mod.rs => models.rs} (100%) delete mode 100644 tests/cli.rs create mode 100644 tests/common/mod.rs create mode 100644 tests/github.rs create mode 100644 tests/help.rs create mode 100644 tests/scan.rs create mode 100644 tests/scan_workflow.rs delete mode 100644 tests/snapshots/cli__noseyparker_github_repos_list_badtoken-3.snap delete mode 100644 tests/snapshots/cli__noseyparker_help_report-3.snap delete mode 100644 tests/snapshots/cli__noseyparker_help_report.snap delete mode 100644 tests/snapshots/cli__noseyparker_help_rules-3.snap delete mode 100644 tests/snapshots/cli__noseyparker_help_rules.snap delete mode 100644 tests/snapshots/cli__noseyparker_help_scan-3.snap delete mode 100644 tests/snapshots/cli__noseyparker_help_scan.snap delete mode 100644 tests/snapshots/cli__noseyparker_help_summarize-3.snap delete mode 100644 tests/snapshots/cli__noseyparker_help_summarize.snap delete mode 100644 tests/snapshots/cli__noseyparker_no_args-2.snap delete mode 100644 tests/snapshots/cli__noseyparker_no_args.snap delete mode 100644 tests/snapshots/cli__noseyparker_scan_emptydir-3.snap delete mode 100644 tests/snapshots/cli__noseyparker_scan_emptydir.snap delete mode 100644 tests/snapshots/cli__noseyparker_scan_secrets1-3.snap delete mode 100644 tests/snapshots/cli__noseyparker_scan_secrets1-4.snap delete mode 100644 tests/snapshots/cli__noseyparker_scan_secrets1-6.snap delete mode 100644 tests/snapshots/cli__noseyparker_scan_secrets1.snap delete mode 100644 tests/snapshots/cli__status.snap delete mode 100644 tests/snapshots/cli__stderr.snap delete mode 100644 tests/snapshots/cli__stdout.snap rename tests/snapshots/{cli__noseyparker_github_repos_list_badtoken-2.snap => github__github_repos_list_noargs-2.snap} (53%) rename tests/snapshots/{cli__noseyparker_github_repos_list_noargs-3.snap => github__github_repos_list_noargs-3.snap} (71%) rename tests/snapshots/{cli__noseyparker_github_repos_list_user_badtoken.snap => github__github_repos_list_noargs.snap} (63%) rename tests/snapshots/{cli__noseyparker_github_repos_list_org_badtoken-2.snap => github__github_repos_list_org_badtoken-2.snap} (53%) rename tests/snapshots/{cli__noseyparker_github_repos_list_org_badtoken-3.snap => github__github_repos_list_org_badtoken-3.snap} (91%) rename tests/snapshots/{cli__noseyparker_github_repos_list_noargs.snap => github__github_repos_list_org_badtoken.snap} (63%) rename tests/snapshots/{cli__noseyparker_github_repos_list_user_badtoken-2.snap => github__github_repos_list_user_badtoken-2.snap} (53%) rename tests/snapshots/{cli__noseyparker_github_repos_list_user_badtoken-3.snap => github__github_repos_list_user_badtoken-3.snap} (91%) rename tests/snapshots/{cli__noseyparker_github_repos_list_badtoken.snap => github__github_repos_list_user_badtoken.snap} (63%) rename tests/snapshots/{cli__noseyparker_help-2.snap => help__help-2.snap} (98%) rename tests/snapshots/{cli__noseyparker_help-3.snap => help__help-3.snap} (56%) rename tests/snapshots/{cli__noseyparker_help.snap => help__help.snap} (65%) rename tests/snapshots/{cli__noseyparker_help_datastore-2.snap => help__help_datastore-2.snap} (97%) rename tests/snapshots/{cli__noseyparker_help_datastore-3.snap => help__help_datastore-3.snap} (56%) rename tests/snapshots/{cli__noseyparker_help_datastore.snap => help__help_datastore.snap} (65%) rename tests/snapshots/{cli__noseyparker_help_github-2.snap => help__help_github-2.snap} (98%) rename tests/snapshots/{cli__noseyparker_help_github_repos-3.snap => help__help_github-3.snap} (56%) rename tests/snapshots/{cli__noseyparker_help_github.snap => help__help_github.snap} (65%) rename tests/snapshots/{cli__noseyparker_help_github_repos-2.snap => help__help_github_repos-2.snap} (98%) rename tests/snapshots/{cli__noseyparker_help_github-3.snap => help__help_github_repos-3.snap} (56%) rename tests/snapshots/{cli__noseyparker_help_github_repos.snap => help__help_github_repos.snap} (65%) create mode 100644 tests/snapshots/help__help_github_repos_short-2.snap create mode 100644 tests/snapshots/help__help_github_repos_short-3.snap create mode 100644 tests/snapshots/help__help_github_repos_short.snap create mode 100644 tests/snapshots/help__help_github_short-2.snap create mode 100644 tests/snapshots/help__help_github_short-3.snap create mode 100644 tests/snapshots/help__help_github_short.snap rename tests/snapshots/{cli__noseyparker_help_report-2.snap => help__help_report-2.snap} (98%) create mode 100644 tests/snapshots/help__help_report-3.snap create mode 100644 tests/snapshots/help__help_report.snap create mode 100644 tests/snapshots/help__help_report_short-2.snap create mode 100644 tests/snapshots/help__help_report_short-3.snap create mode 100644 tests/snapshots/help__help_report_short.snap rename tests/snapshots/{cli__noseyparker_help_rules-2.snap => help__help_rules-2.snap} (97%) create mode 100644 tests/snapshots/help__help_rules-3.snap create mode 100644 tests/snapshots/help__help_rules.snap rename tests/snapshots/{cli__noseyparker_help_scan-2.snap => help__help_scan-2.snap} (99%) create mode 100644 tests/snapshots/help__help_scan-3.snap create mode 100644 tests/snapshots/help__help_scan.snap create mode 100644 tests/snapshots/help__help_scan_short-2.snap create mode 100644 tests/snapshots/help__help_scan_short-3.snap create mode 100644 tests/snapshots/help__help_scan_short.snap create mode 100644 tests/snapshots/help__help_short-2.snap create mode 100644 tests/snapshots/help__help_short-3.snap create mode 100644 tests/snapshots/help__help_short.snap rename tests/snapshots/{cli__noseyparker_help_summarize-2.snap => help__help_summarize-2.snap} (98%) create mode 100644 tests/snapshots/help__help_summarize-3.snap create mode 100644 tests/snapshots/help__help_summarize.snap create mode 100644 tests/snapshots/help__help_summarize_short-2.snap create mode 100644 tests/snapshots/help__help_summarize_short-3.snap create mode 100644 tests/snapshots/help__help_summarize_short.snap rename tests/snapshots/{cli__noseyparker_github_repos_list_noargs-2.snap => help__no_args-2.snap} (56%) rename tests/snapshots/{cli__noseyparker_no_args-3.snap => help__no_args-3.snap} (97%) rename tests/snapshots/{cli__noseyparker_github_repos_list_org_badtoken.snap => help__no_args.snap} (65%) rename tests/snapshots/{cli__noseyparker_scan_secrets1-2.snap => scan_workflow__scan_secrets1-2.snap} (89%) create mode 100644 tests/snapshots/scan_workflow__scan_secrets1-3.snap create mode 100644 tests/snapshots/scan_workflow__scan_secrets1-4.snap rename tests/snapshots/{cli__noseyparker_scan_secrets1-5.snap => scan_workflow__scan_secrets1-5.snap} (89%) create mode 100644 tests/snapshots/scan_workflow__scan_secrets1-6.snap rename tests/snapshots/{cli__noseyparker_scan_secrets1-7.snap => scan_workflow__scan_secrets1-7.snap} (96%) create mode 100644 tests/snapshots/scan_workflow__scan_secrets1.snap diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23a51bb49..7888a99cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,6 +142,11 @@ jobs: run: cargo build --release --verbose --locked - name: Run tests + env: + # We use the GitHub Actions automatic token when running tests, to avoid + # spurious failures from rate limiting when testing Nosey Parker's github + # enumeration capabilities. + - NP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: cargo test --release --verbose --locked - name: Check rules diff --git a/src/github/mod.rs b/src/github.rs similarity index 100% rename from src/github/mod.rs rename to src/github.rs diff --git a/src/github/models/mod.rs b/src/github/models.rs similarity index 100% rename from src/github/models/mod.rs rename to src/github/models.rs diff --git a/tests/cli.rs b/tests/cli.rs deleted file mode 100644 index 90c6056fa..000000000 --- a/tests/cli.rs +++ /dev/null @@ -1,419 +0,0 @@ -use assert_cmd::prelude::*; -use assert_fs::prelude::*; -use assert_fs::{fixture::ChildPath, TempDir}; -use indoc::indoc; -use insta::{assert_display_snapshot, assert_snapshot, assert_json_snapshot, with_settings}; -use lazy_static::lazy_static; -// use predicates::prelude::*; -use predicates::str::RegexPredicate; -use std::path::Path; -use std::process::Command; - -// ------------------------------------------------------------------------------------------------- -// Utilities -// ------------------------------------------------------------------------------------------------- - -/// Use `insta` to do snapshot testing against a command's exit code, stdout, and stderr. -/// -/// The given expression should be an `assert_cmd::assert::Assert`. -macro_rules! assert_cmd_snapshot { - ( $cmd:expr ) => { - let cmd = $cmd; - let output = cmd.get_output(); - let status = output.status; - assert_display_snapshot!(status); - let stdout = String::from_utf8(output.stdout.clone()).unwrap(); - assert_snapshot!(stdout); - let stderr = String::from_utf8(output.stderr.clone()).unwrap(); - assert_snapshot!(stderr); - }; -} - -/// Build a `Command` for the `noseyparker` crate binary with variadic command-line arguments. -/// -/// The arguments can be anything that is allowed by `Command::arg`. -macro_rules! noseyparker { - ( $( $arg:expr ),* ) => { - { - let mut cmd = noseyparker(); - $( - cmd.arg($arg); - )* - cmd - } - } -} - -/// Build an `assert_cmd::assert::Assert` by calling `noseyparker!(args).assert().success()`. -macro_rules! noseyparker_success { - ( $( $arg:expr ),* ) => { noseyparker!($( $arg ),*).assert().success() } -} - -/// Build an `assert_cmd::assert::Assert` by calling `noseyparker!(args).assert().failure()`. -macro_rules! noseyparker_failure { - ( $( $arg:expr ),* ) => { noseyparker!($( $arg ),*).assert().failure() } -} - -lazy_static! { - static ref NOSEYPARKER: escargot::CargoRun = escargot::CargoBuild::new() - .bin("noseyparker") - .run() - .expect("noseyparker should be available"); - - // We could use this to write tests for specific feature configurations: - /* - static ref NOSEYPARKER_RULE_PROFILING: escargot::CargoRun = escargot::CargoBuild::new() - .bin("noseyparker") - .no_default_features() - .features("rule_profiling") - .run() - .expect("noseyparker with rule_profiling should be available"); - */ -} - -/// Build a `Command` for the `noseyparker` crate binary. -pub fn noseyparker() -> Command { - // Command::cargo_bin("noseyparker").expect("noseyparker should be executable") - NOSEYPARKER.command() -} - -/// Create a `RegexPredicate` from the given pattern. -pub fn is_match(pat: &str) -> RegexPredicate { - predicates::str::is_match(pat).expect("pattern should compile") -} - -/// Create a `RegexPredicate` for matching a scan stats output message from Nosey Parker. -pub fn match_scan_stats( - num_bytes: &str, - num_blobs: u64, - new_matches: u64, - total_matches: u64, -) -> RegexPredicate { - is_match(&format!( - r"(?m)^Scanned {} from {} blobs in .*; {}/{} new matches$", - num_bytes, num_blobs, new_matches, total_matches - )) -} - -/// Create a `RegexPredicate` for matching a "nothing was scanned" scan stats output message from -/// Nosey Parker. -pub fn match_nothing_scanned() -> RegexPredicate { - match_scan_stats("0B", 0, 0, 0) -} - -/// A type to represent a mock scanning environment for testing Nosey Parker. -pub struct ScanEnv { - pub root: TempDir, - pub datastore: ChildPath, -} - -impl ScanEnv { - /// Create a new mock scanning environment. - pub fn new() -> Self { - let root = TempDir::new().expect("should be able to create tempdir"); - let datastore = root.child("datastore"); - assert!(!datastore.exists()); - - Self { root, datastore } - } - - /// Create an empty file within this mock scanning environment with the given name. - pub fn input_file(&self, name: &str) -> ChildPath { - let input = self.root.child(name); - input.touch().expect("should be able to write input file"); - assert!(input.is_file()); - input - } - - /// Create an input file within this mock scanning environment with the given name. - /// The created input file will have content containing a fake AWS key that should be detected. - pub fn input_file_with_secret(&self, name: &str) -> ChildPath { - let input = self.root.child(name); - input.touch().expect("should be able to write input file"); - assert!(input.is_file()); - let contents = indoc! {r#" - # This is fake configuration data - USERNAME=the_dude - AWS_KEY=AKIADEADBEEFDEADBEEF - "#}; - input - .write_str(contents) - .expect("should be able to write input file contents"); - input - } - - /// Create an empty directory within this mock scanning environment with the given name. - pub fn input_dir(&self, name: &str) -> ChildPath { - let input = self.root.child(name); - input - .create_dir_all() - .expect("should be able to create input directory"); - assert!(input.is_dir()); - input - } - - /// Create a name for a child entry within this mock scanning environment. - /// - /// The filesystem is not touched by this function; this merely produces a `ChildPath`. - pub fn child(&self, name: &str) -> ChildPath { - self.root.child(name) - } - - /// Get the path to the Nosey Parker datastore directory within this mock scanning environment. - pub fn dspath(&self) -> &Path { - self.datastore.path() - } -} - -// ------------------------------------------------------------------------------------------------- -// Tests -// ------------------------------------------------------------------------------------------------- - -#[test] -fn test_noseyparker_no_args() { - assert_cmd_snapshot!(noseyparker!().assert().failure()); -} - -#[test] -fn test_noseyparker_help() { - assert_cmd_snapshot!(noseyparker_success!("help")); -} - -#[test] -fn test_noseyparker_help_scan() { - with_settings!({ - filters => vec![ - (r"(?m)(scanning jobs\s+)\[default: \d+\]", r"$1[default: DEFAULT]") - ], - }, { - assert_cmd_snapshot!(noseyparker_success!("help", "scan")); - }); -} - -#[test] -fn test_noseyparker_help_summarize() { - assert_cmd_snapshot!(noseyparker_success!("help", "summarize")); -} - -#[test] -fn test_noseyparker_help_report() { - assert_cmd_snapshot!(noseyparker_success!("help", "report")); -} - -#[test] -fn test_noseyparker_help_datastore() { - assert_cmd_snapshot!(noseyparker_success!("help", "datastore")); -} - -#[test] -fn test_noseyparker_help_rules() { - assert_cmd_snapshot!(noseyparker_success!("help", "rules")); -} - -#[test] -fn test_noseyparker_help_github() { - assert_cmd_snapshot!(noseyparker_success!("help", "github")); -} - -#[test] -fn test_noseyparker_help_github_repos() { - assert_cmd_snapshot!(noseyparker_success!("help", "github", "repos")); -} - -#[test] -fn test_noseyparker_github_repos_list_noargs() { - assert_cmd_snapshot!(noseyparker_failure!("github", "repos", "list")); -} - -#[test] -fn test_noseyparker_github_repos_list_org_badtoken() { - let cmd = noseyparker() - .args(&["github", "repos", "list", "--org", "praetorian-inc"]) - .env("NP_GITHUB_TOKEN", "hahabogus") - .assert() - .failure(); - assert_cmd_snapshot!(cmd); -} - -#[test] -fn test_noseyparker_github_repos_list_user_badtoken() { - let cmd = noseyparker() - .args(&["github", "repos", "list", "--user", "octocat"]) - .env("NP_GITHUB_TOKEN", "hahabogus") - .assert() - .failure(); - assert_cmd_snapshot!(cmd); -} - -// XXX These are disabled for now due to GitHub Actions rate limiting troubles. -// FIXME: Figure out how to plumb an access token to these tests, then re-enable them. -/* -#[test] -fn test_noseyparker_github_repos_list_user_unauthenticated_human_format() { - noseyparker_success!("github", "repos", "list", "--user", "octocat") - .stdout(predicates::str::contains("https://github.com/octocat/Spoon-Knife.git")) - .stderr(predicates::str::is_empty()); -} - -#[test] -fn test_noseyparker_github_repos_list_user_unauthenticated_jsonl_format() { - noseyparker_success!("github", "repos", "list", "--user", "octocat", "--format", "jsonl") - .stdout(predicates::str::contains("\"https://github.com/octocat/Spoon-Knife.git\"\n")) - .stderr(predicates::str::is_empty()); -} - -#[test] -fn test_noseyparker_github_repos_list_user_unauthenticated_json_format() { - let cmd = noseyparker_success!("github", "repos", "list", "--user", "octocat", "--format", "json") - .stderr(predicates::str::is_empty()); - let output = &cmd.get_output().stdout; - let json_parsed: Vec = serde_json::from_slice(output).expect("output should be well-formed JSON"); - assert!(json_parsed.contains(&String::from("https://github.com/octocat/Spoon-Knife.git")), - "JSON output does not contain https://github.com/octocat/Spoon-Knife.git: {json_parsed:?}"); -} -*/ - -// XXX additional `github repo list` cases to test: -// -// - multiple users / orgs -// - duplicated users / orgs: output list should be deduped -// - using a real access token - -#[test] -fn test_noseyparker_scan_emptydir() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_dir("empty_dir"); - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) - .stdout(match_nothing_scanned()); -} - -#[test] -fn test_noseyparker_scan_datastore_argorder() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_dir("empty_dir"); - noseyparker_success!("scan", input.path(), "--datastore", scan_env.dspath()) - .stdout(match_nothing_scanned()); -} - -#[test] -fn test_noseyparker_scan_datastore_short() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_dir("empty_dir"); - noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) - .stdout(match_nothing_scanned()); -} - -#[test] -fn test_noseyparker_scan_datastore_envvar() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_dir("empty_dir"); - noseyparker!("scan", input.path()) - .env("NP_DATASTORE", scan_env.dspath()) - .assert() - .success() - .stdout(match_nothing_scanned()); -} - -#[test] -fn test_noseyparker_scan_emptyfile() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_file("empty_file"); - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) - .stdout(match_scan_stats("0B", 1, 0, 0)); -} - -#[test] -fn test_noseyparker_scan_emptyfiles() { - let scan_env = ScanEnv::new(); - let input1 = scan_env.input_file("empty_file1"); - let input2 = scan_env.input_file("empty_file2"); - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input1.path(), input2.path()) - .stdout(match_scan_stats("0B", 2, 0, 0)); -} - -#[test] -fn test_noseyparker_scan_file_symlink() { - let scan_env = ScanEnv::new(); - let empty_file = scan_env.input_file("empty_file"); - let input = scan_env.child("empty_file_link"); - input.symlink_to_file(empty_file).unwrap(); - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) - .stdout(match_nothing_scanned()); -} - -#[test] -fn test_noseyparker_scan_file_maxsize() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_file("bigfile.dat"); - input.write_binary(&[b'a'; 1024 * 1024 * 10]).unwrap(); - - // By default the input file gets scanned - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) - .stdout(match_scan_stats("10.00 MiB", 1, 0, 0)); - - // With a restricted max file size, the file is not scanned - noseyparker_success!( - "scan", - "--datastore", - scan_env.dspath(), - input.path(), - "--max-file-size", - "5" - ) - .stdout(match_nothing_scanned()); - - // Also check for alternatively-spelled versions of a couple arguments - noseyparker_success!( - "scan", - format!("-d={}", scan_env.dspath().display()), - "--max-file-size=5.00", - input.path() - ) - .stdout(match_nothing_scanned()); -} - -#[cfg(unix)] -#[test] -fn test_noseyparker_scan_unreadable_file() { - use std::fs::{File, Permissions}; - use std::os::unix::fs::PermissionsExt; - - let scan_env = ScanEnv::new(); - let input = scan_env.input_file_with_secret("input.txt"); - // n.b. file value explicitly unnamed so it gets dropped - File::open(input.path()) - .unwrap() - .set_permissions(Permissions::from_mode(0o000)) - .unwrap(); - assert!(std::fs::read_to_string(input.path()).is_err()); - - noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) - .stdout(is_match("ERROR.*: Failed to load blob from .*: Permission denied")) - .stdout(match_nothing_scanned()); -} - -#[test] -fn test_noseyparker_scan_secrets1() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_file_with_secret("input.txt"); - - noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) - .stdout(match_scan_stats("81B", 1, 1, 1)); - - assert_cmd_snapshot!(noseyparker_success!("summarize", "-d", scan_env.dspath())); - - with_settings!({ - filters => vec![ - (r"(?m)^(\s*File: ).*$", r"$1 ") - ], - }, { - assert_cmd_snapshot!(noseyparker_success!("report", "-d", scan_env.dspath())); - }); - - - let cmd = noseyparker_success!("report", "-d", scan_env.dspath(), "--format=json"); - let json_output: serde_json::Value = serde_json::from_slice(&cmd.get_output().stdout).unwrap(); - assert_json_snapshot!(json_output, { - "[].matches[].provenance.path" => "/input.txt" - }); -} diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 000000000..22671c7d0 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,170 @@ +//! Integration Test Utilities and Common Code + +#![allow(dead_code)] + +use indoc::indoc; +use lazy_static::lazy_static; + +pub use assert_cmd::prelude::*; +pub use assert_fs::prelude::*; +pub use assert_fs::{fixture::ChildPath, TempDir}; +pub use insta::{assert_display_snapshot, assert_json_snapshot, assert_snapshot, with_settings}; +pub use predicates::str::RegexPredicate; +pub use std::path::Path; +pub use std::process::Command; + +/// Use `insta` to do snapshot testing against a command's exit code, stdout, and stderr. +/// +/// The given expression should be an `assert_cmd::assert::Assert`. +#[macro_export] +macro_rules! assert_cmd_snapshot { + ( $cmd:expr ) => { + let cmd = $cmd; + let output = cmd.get_output(); + let status = output.status; + assert_display_snapshot!(status); + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + assert_snapshot!(stdout); + let stderr = String::from_utf8(output.stderr.clone()).unwrap(); + assert_snapshot!(stderr); + }; +} + +/// Build a `Command` for the `noseyparker` crate binary with variadic command-line arguments. +/// +/// The arguments can be anything that is allowed by `Command::arg`. +#[macro_export] +macro_rules! noseyparker { + ( $( $arg:expr ),* ) => { + { + let mut cmd = common::noseyparker(); + $( + cmd.arg($arg); + )* + cmd + } + } +} + +/// Build an `assert_cmd::assert::Assert` by calling `noseyparker!(args).assert().success()`. +#[macro_export] +macro_rules! noseyparker_success { + ( $( $arg:expr ),* ) => { noseyparker!($( $arg ),*).assert().success() } +} + +/// Build an `assert_cmd::assert::Assert` by calling `noseyparker!(args).assert().failure()`. +#[macro_export] +macro_rules! noseyparker_failure { + ( $( $arg:expr ),* ) => { noseyparker!($( $arg ),*).assert().failure() } +} + +lazy_static! { + static ref NOSEYPARKER: escargot::CargoRun = escargot::CargoBuild::new() + .bin("noseyparker") + .run() + .expect("noseyparker should be available"); + + // We could use this to write tests for specific feature configurations: + /* + static ref NOSEYPARKER_RULE_PROFILING: escargot::CargoRun = escargot::CargoBuild::new() + .bin("noseyparker") + .no_default_features() + .features("rule_profiling") + .run() + .expect("noseyparker with rule_profiling should be available"); + */ +} + +/// Build a `Command` for the `noseyparker` crate binary. +pub fn noseyparker() -> Command { + // Command::cargo_bin("noseyparker").expect("noseyparker should be executable") + NOSEYPARKER.command() +} + +/// Create a `RegexPredicate` from the given pattern. +pub fn is_match(pat: &str) -> RegexPredicate { + predicates::str::is_match(pat).expect("pattern should compile") +} + +/// Create a `RegexPredicate` for matching a scan stats output message from Nosey Parker. +pub fn match_scan_stats( + num_bytes: &str, + num_blobs: u64, + new_matches: u64, + total_matches: u64, +) -> RegexPredicate { + is_match(&format!( + r"(?m)^Scanned {} from {} blobs in .*; {}/{} new matches$", + num_bytes, num_blobs, new_matches, total_matches + )) +} + +/// Create a `RegexPredicate` for matching a "nothing was scanned" scan stats output message from +/// Nosey Parker. +pub fn match_nothing_scanned() -> RegexPredicate { + match_scan_stats("0B", 0, 0, 0) +} + +/// A type to represent a mock scanning environment for testing Nosey Parker. +pub struct ScanEnv { + pub root: TempDir, + pub datastore: ChildPath, +} + +impl ScanEnv { + /// Create a new mock scanning environment. + pub fn new() -> Self { + let root = TempDir::new().expect("should be able to create tempdir"); + let datastore = root.child("datastore"); + assert!(!datastore.exists()); + + Self { root, datastore } + } + + /// Create an empty file within this mock scanning environment with the given name. + pub fn input_file(&self, name: &str) -> ChildPath { + let input = self.root.child(name); + input.touch().expect("should be able to write input file"); + assert!(input.is_file()); + input + } + + /// Create an input file within this mock scanning environment with the given name. + /// The created input file will have content containing a fake AWS key that should be detected. + pub fn input_file_with_secret(&self, name: &str) -> ChildPath { + let input = self.root.child(name); + input.touch().expect("should be able to write input file"); + assert!(input.is_file()); + let contents = indoc! {r#" + # This is fake configuration data + USERNAME=the_dude + AWS_KEY=AKIADEADBEEFDEADBEEF + "#}; + input + .write_str(contents) + .expect("should be able to write input file contents"); + input + } + + /// Create an empty directory within this mock scanning environment with the given name. + pub fn input_dir(&self, name: &str) -> ChildPath { + let input = self.root.child(name); + input + .create_dir_all() + .expect("should be able to create input directory"); + assert!(input.is_dir()); + input + } + + /// Create a name for a child entry within this mock scanning environment. + /// + /// The filesystem is not touched by this function; this merely produces a `ChildPath`. + pub fn child(&self, name: &str) -> ChildPath { + self.root.child(name) + } + + /// Get the path to the Nosey Parker datastore directory within this mock scanning environment. + pub fn dspath(&self) -> &Path { + self.datastore.path() + } +} diff --git a/tests/github.rs b/tests/github.rs new file mode 100644 index 000000000..1a3ece56b --- /dev/null +++ b/tests/github.rs @@ -0,0 +1,83 @@ +//! Tests for Nosey Parker `github` command + +mod common; +use common::*; + +#[test] +fn github_repos_list_noargs() { + assert_cmd_snapshot!(noseyparker_failure!("github", "repos", "list")); +} + +#[test] +fn github_repos_list_org_badtoken() { + let cmd = noseyparker() + .args(&["github", "repos", "list", "--org", "praetorian-inc"]) + .env("NP_GITHUB_TOKEN", "hahabogus") + .assert() + .failure(); + assert_cmd_snapshot!(cmd); +} + +#[test] +fn github_repos_list_user_badtoken() { + let cmd = noseyparker() + .args(&["github", "repos", "list", "--user", "octocat"]) + .env("NP_GITHUB_TOKEN", "hahabogus") + .assert() + .failure(); + assert_cmd_snapshot!(cmd); +} + + +// XXX Note: `octocat` is not a user under our control; it's a kind of test account owned by GitHub. +// We are assuming that the `octocat` user's list of repositories will always include `Spoon-Knife`. + +// XXX Note: the following test cases make actual GitHub requests and may fail due to rate limiting +// issues when not using a token. +// +// To avoid flaky tests, we have these tests use a token from the environment when `CI=1` (set in +// GitHub Actions), and use no token otherwise. +fn handle_github_token(cmd: &mut Command) { + if std::env::var("CI").is_ok() { + assert!(std::env::var("NP_GITHUB_TOKEN").is_ok()); + } else { + cmd.env_remove("NP_GITHUB_TOKEN"); + } +} + +#[test] +fn github_repos_list_user_human_format() { + let mut cmd = noseyparker!("github", "repos", "list", "--user", "octocat"); + handle_github_token(&mut cmd); + cmd + .assert() + .success() + .stdout(predicates::str::contains("https://github.com/octocat/Spoon-Knife.git")) + .stderr(predicates::str::is_empty()); +} + +#[test] +fn github_repos_list_user_jsonl_format() { + let mut cmd = noseyparker!("github", "repos", "list", "--user", "octocat", "--format", "jsonl"); + handle_github_token(&mut cmd); + cmd + .assert() + .success() + .stdout(predicates::str::contains("\"https://github.com/octocat/Spoon-Knife.git\"\n")) + .stderr(predicates::str::is_empty()); +} + +#[test] +fn github_repos_list_user_json_format() { + let mut cmd = noseyparker!("github", "repos", "list", "--user", "octocat", "--format", "json"); + handle_github_token(&mut cmd); + let cmd = cmd + .assert() + .success() + .stderr(predicates::str::is_empty()); + + let output = &cmd.get_output().stdout; + let json_parsed: Vec = serde_json::from_slice(output).expect("output should be well-formed JSON"); + assert!(json_parsed.contains(&String::from("https://github.com/octocat/Spoon-Knife.git")), + "JSON output does not contain https://github.com/octocat/Spoon-Knife.git: {json_parsed:?}"); +} diff --git a/tests/help.rs b/tests/help.rs new file mode 100644 index 000000000..fa2f4f788 --- /dev/null +++ b/tests/help.rs @@ -0,0 +1,92 @@ +//! Tests for Nosey Parker `help` functionality + +mod common; +use common::*; + +#[test] +fn no_args() { + assert_cmd_snapshot!(noseyparker!().assert().failure()); +} + +#[test] +fn help() { + assert_cmd_snapshot!(noseyparker_success!("help")); +} + +#[test] +fn help_short() { + assert_cmd_snapshot!(noseyparker_success!("-h")); +} + +#[test] +fn help_scan() { + with_settings!({ + filters => vec![ + (r"(?m)(scanning jobs\s+)\[default: \d+\]", r"$1[default: DEFAULT]") + ], + }, { + assert_cmd_snapshot!(noseyparker_success!("help", "scan")); + }); +} + +#[test] +fn help_scan_short() { + with_settings!({ + filters => vec![ + (r"(?m)(scanning jobs\s+)\[default: \d+\]", r"$1[default: DEFAULT]") + ], + }, { + assert_cmd_snapshot!(noseyparker_success!("scan", "-h")); + }); +} + +#[test] +fn help_summarize() { + assert_cmd_snapshot!(noseyparker_success!("help", "summarize")); +} + +#[test] +fn help_summarize_short() { + assert_cmd_snapshot!(noseyparker_success!("summarize", "-h")); +} + +#[test] +fn help_report() { + assert_cmd_snapshot!(noseyparker_success!("help", "report")); +} + +#[test] +fn help_report_short() { + assert_cmd_snapshot!(noseyparker_success!("report", "-h")); +} + + +#[test] +fn help_datastore() { + assert_cmd_snapshot!(noseyparker_success!("help", "datastore")); +} + +#[test] +fn help_rules() { + assert_cmd_snapshot!(noseyparker_success!("help", "rules")); +} + +#[test] +fn help_github() { + assert_cmd_snapshot!(noseyparker_success!("help", "github")); +} + +#[test] +fn help_github_short() { + assert_cmd_snapshot!(noseyparker_success!("github", "-h")); +} + +#[test] +fn help_github_repos() { + assert_cmd_snapshot!(noseyparker_success!("help", "github", "repos")); +} + +#[test] +fn help_github_repos_short() { + assert_cmd_snapshot!(noseyparker_success!("github", "repos", "-h")); +} diff --git a/tests/scan.rs b/tests/scan.rs new file mode 100644 index 000000000..bb4c44968 --- /dev/null +++ b/tests/scan.rs @@ -0,0 +1,117 @@ +//! Tests for Nosey Parker `scan` command + +mod common; +use common::*; + +#[test] +fn scan_emptydir() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_dir("empty_dir"); + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_datastore_argorder() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_dir("empty_dir"); + noseyparker_success!("scan", input.path(), "--datastore", scan_env.dspath()) + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_datastore_short() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_dir("empty_dir"); + noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_datastore_envvar() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_dir("empty_dir"); + noseyparker!("scan", input.path()) + .env("NP_DATASTORE", scan_env.dspath()) + .assert() + .success() + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_emptyfile() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_file("empty_file"); + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) + .stdout(match_scan_stats("0B", 1, 0, 0)); +} + +#[test] +fn scan_emptyfiles() { + let scan_env = ScanEnv::new(); + let input1 = scan_env.input_file("empty_file1"); + let input2 = scan_env.input_file("empty_file2"); + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input1.path(), input2.path()) + .stdout(match_scan_stats("0B", 2, 0, 0)); +} + +#[test] +fn scan_file_symlink() { + let scan_env = ScanEnv::new(); + let empty_file = scan_env.input_file("empty_file"); + let input = scan_env.child("empty_file_link"); + input.symlink_to_file(empty_file).unwrap(); + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_file_maxsize() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_file("bigfile.dat"); + input.write_binary(&[b'a'; 1024 * 1024 * 10]).unwrap(); + + // By default the input file gets scanned + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) + .stdout(match_scan_stats("10.00 MiB", 1, 0, 0)); + + // With a restricted max file size, the file is not scanned + noseyparker_success!( + "scan", + "--datastore", + scan_env.dspath(), + input.path(), + "--max-file-size", + "5" + ) + .stdout(match_nothing_scanned()); + + // Also check for alternatively-spelled versions of a couple arguments + noseyparker_success!( + "scan", + format!("-d={}", scan_env.dspath().display()), + "--max-file-size=5.00", + input.path() + ) + .stdout(match_nothing_scanned()); +} + +#[cfg(unix)] +#[test] +fn scan_unreadable_file() { + use std::fs::{File, Permissions}; + use std::os::unix::fs::PermissionsExt; + + let scan_env = ScanEnv::new(); + let input = scan_env.input_file_with_secret("input.txt"); + // n.b. file value explicitly unnamed so it gets dropped + File::open(input.path()) + .unwrap() + .set_permissions(Permissions::from_mode(0o000)) + .unwrap(); + assert!(std::fs::read_to_string(input.path()).is_err()); + + noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) + .stdout(is_match("ERROR.*: Failed to load blob from .*: Permission denied")) + .stdout(match_nothing_scanned()); +} diff --git a/tests/scan_workflow.rs b/tests/scan_workflow.rs new file mode 100644 index 000000000..448782440 --- /dev/null +++ b/tests/scan_workflow.rs @@ -0,0 +1,30 @@ +//! Tests for a complete Nosey Parker scanning workflow + +mod common; +use common::*; + +#[test] +fn scan_secrets1() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_file_with_secret("input.txt"); + + noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) + .stdout(match_scan_stats("81B", 1, 1, 1)); + + assert_cmd_snapshot!(noseyparker_success!("summarize", "-d", scan_env.dspath())); + + with_settings!({ + filters => vec![ + (r"(?m)^(\s*File: ).*$", r"$1 ") + ], + }, { + assert_cmd_snapshot!(noseyparker_success!("report", "-d", scan_env.dspath())); + }); + + + let cmd = noseyparker_success!("report", "-d", scan_env.dspath(), "--format=json"); + let json_output: serde_json::Value = serde_json::from_slice(&cmd.get_output().stdout).unwrap(); + assert_json_snapshot!(json_output, { + "[].matches[].provenance.path" => "/input.txt" + }); +} diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_badtoken-3.snap b/tests/snapshots/cli__noseyparker_github_repos_list_badtoken-3.snap deleted file mode 100644 index c4a495987..000000000 --- a/tests/snapshots/cli__noseyparker_github_repos_list_badtoken-3.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: tests/cli.rs -expression: stderr ---- -Error: error making request: HTTP status client error (401 Unauthorized) for url (https://api.github.com/rate_limit) - -Caused by: - HTTP status client error (401 Unauthorized) for url (https://api.github.com/rate_limit) - diff --git a/tests/snapshots/cli__noseyparker_help_report-3.snap b/tests/snapshots/cli__noseyparker_help_report-3.snap deleted file mode 100644 index 09fa0f492..000000000 --- a/tests/snapshots/cli__noseyparker_help_report-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: stderr ---- - diff --git a/tests/snapshots/cli__noseyparker_help_report.snap b/tests/snapshots/cli__noseyparker_help_report.snap deleted file mode 100644 index 66b7159c9..000000000 --- a/tests/snapshots/cli__noseyparker_help_report.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_rules-3.snap b/tests/snapshots/cli__noseyparker_help_rules-3.snap deleted file mode 100644 index 09fa0f492..000000000 --- a/tests/snapshots/cli__noseyparker_help_rules-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: stderr ---- - diff --git a/tests/snapshots/cli__noseyparker_help_rules.snap b/tests/snapshots/cli__noseyparker_help_rules.snap deleted file mode 100644 index 66b7159c9..000000000 --- a/tests/snapshots/cli__noseyparker_help_rules.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_scan-3.snap b/tests/snapshots/cli__noseyparker_help_scan-3.snap deleted file mode 100644 index 09fa0f492..000000000 --- a/tests/snapshots/cli__noseyparker_help_scan-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: stderr ---- - diff --git a/tests/snapshots/cli__noseyparker_help_scan.snap b/tests/snapshots/cli__noseyparker_help_scan.snap deleted file mode 100644 index 66b7159c9..000000000 --- a/tests/snapshots/cli__noseyparker_help_scan.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_summarize-3.snap b/tests/snapshots/cli__noseyparker_help_summarize-3.snap deleted file mode 100644 index 09fa0f492..000000000 --- a/tests/snapshots/cli__noseyparker_help_summarize-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: stderr ---- - diff --git a/tests/snapshots/cli__noseyparker_help_summarize.snap b/tests/snapshots/cli__noseyparker_help_summarize.snap deleted file mode 100644 index 66b7159c9..000000000 --- a/tests/snapshots/cli__noseyparker_help_summarize.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_no_args-2.snap b/tests/snapshots/cli__noseyparker_no_args-2.snap deleted file mode 100644 index 02f5417a6..000000000 --- a/tests/snapshots/cli__noseyparker_no_args-2.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: stdout ---- - diff --git a/tests/snapshots/cli__noseyparker_no_args.snap b/tests/snapshots/cli__noseyparker_no_args.snap deleted file mode 100644 index 7749ce275..000000000 --- a/tests/snapshots/cli__noseyparker_no_args.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: status ---- -exit status: 2 diff --git a/tests/snapshots/cli__noseyparker_scan_emptydir-3.snap b/tests/snapshots/cli__noseyparker_scan_emptydir-3.snap deleted file mode 100644 index 09fa0f492..000000000 --- a/tests/snapshots/cli__noseyparker_scan_emptydir-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: stderr ---- - diff --git a/tests/snapshots/cli__noseyparker_scan_emptydir.snap b/tests/snapshots/cli__noseyparker_scan_emptydir.snap deleted file mode 100644 index 66b7159c9..000000000 --- a/tests/snapshots/cli__noseyparker_scan_emptydir.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_scan_secrets1-3.snap b/tests/snapshots/cli__noseyparker_scan_secrets1-3.snap deleted file mode 100644 index 09fa0f492..000000000 --- a/tests/snapshots/cli__noseyparker_scan_secrets1-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: stderr ---- - diff --git a/tests/snapshots/cli__noseyparker_scan_secrets1-4.snap b/tests/snapshots/cli__noseyparker_scan_secrets1-4.snap deleted file mode 100644 index 66b7159c9..000000000 --- a/tests/snapshots/cli__noseyparker_scan_secrets1-4.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_scan_secrets1-6.snap b/tests/snapshots/cli__noseyparker_scan_secrets1-6.snap deleted file mode 100644 index 09fa0f492..000000000 --- a/tests/snapshots/cli__noseyparker_scan_secrets1-6.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: stderr ---- - diff --git a/tests/snapshots/cli__noseyparker_scan_secrets1.snap b/tests/snapshots/cli__noseyparker_scan_secrets1.snap deleted file mode 100644 index 66b7159c9..000000000 --- a/tests/snapshots/cli__noseyparker_scan_secrets1.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/cli__status.snap b/tests/snapshots/cli__status.snap deleted file mode 100644 index 9baab6eb8..000000000 --- a/tests/snapshots/cli__status.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: output.status ---- -exit status: 0 diff --git a/tests/snapshots/cli__stderr.snap b/tests/snapshots/cli__stderr.snap deleted file mode 100644 index 942c7ab3a..000000000 --- a/tests/snapshots/cli__stderr.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/cli.rs -expression: "String::from_utf8(output.stderr.clone()).unwrap()" ---- - diff --git a/tests/snapshots/cli__stdout.snap b/tests/snapshots/cli__stdout.snap deleted file mode 100644 index f3b3964f6..000000000 --- a/tests/snapshots/cli__stdout.snap +++ /dev/null @@ -1,56 +0,0 @@ ---- -source: tests/cli.rs -expression: "String::from_utf8(output.stdout.clone()).unwrap()" ---- -Find secrets and sensitive information in textual data - -Usage: noseyparker [OPTIONS] - -Commands: - scan - Scan content for secrets - summarize - Summarize scan findings - report - Report detailed scan findings - github - Query GitHub - datastore - Manage datastores - rules - Manage rules - help - Print this message or the help of the given subcommand(s) - -Options: - -h, --help - Print help information (use `-h` for a summary) - - -V, --version - Print version information - -Global Options: - -v, --verbose... - Enable verbose output - - This can be repeated up to 3 times to enable successively more output. - - --color - Enable or disable colored output - - When this is "auto", colors are enabled when stdout is a tty. - - If the `NO_COLOR` environment variable is set, it takes precedence and is equivalent to - `--color=never`. - - [default: auto] - [possible values: auto, never, always] - - --progress - Enable or disable progress bars - - When this is "auto", progress bars are enabled when stderr is a tty. - - [default: auto] - [possible values: auto, never, always] - diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_badtoken-2.snap b/tests/snapshots/github__github_repos_list_noargs-2.snap similarity index 53% rename from tests/snapshots/cli__noseyparker_github_repos_list_badtoken-2.snap rename to tests/snapshots/github__github_repos_list_noargs-2.snap index 02f5417a6..2caca4ffb 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_badtoken-2.snap +++ b/tests/snapshots/github__github_repos_list_noargs-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/github.rs expression: stdout --- diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_noargs-3.snap b/tests/snapshots/github__github_repos_list_noargs-3.snap similarity index 71% rename from tests/snapshots/cli__noseyparker_github_repos_list_noargs-3.snap rename to tests/snapshots/github__github_repos_list_noargs-3.snap index 34ed26cfa..b407bc1c9 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_noargs-3.snap +++ b/tests/snapshots/github__github_repos_list_noargs-3.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/github.rs expression: stderr --- Error: No repositories specified diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_user_badtoken.snap b/tests/snapshots/github__github_repos_list_noargs.snap similarity index 63% rename from tests/snapshots/cli__noseyparker_github_repos_list_user_badtoken.snap rename to tests/snapshots/github__github_repos_list_noargs.snap index 7749ce275..2ee2c85f0 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_user_badtoken.snap +++ b/tests/snapshots/github__github_repos_list_noargs.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/github.rs expression: status --- exit status: 2 diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_org_badtoken-2.snap b/tests/snapshots/github__github_repos_list_org_badtoken-2.snap similarity index 53% rename from tests/snapshots/cli__noseyparker_github_repos_list_org_badtoken-2.snap rename to tests/snapshots/github__github_repos_list_org_badtoken-2.snap index 02f5417a6..2caca4ffb 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_org_badtoken-2.snap +++ b/tests/snapshots/github__github_repos_list_org_badtoken-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/github.rs expression: stdout --- diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_org_badtoken-3.snap b/tests/snapshots/github__github_repos_list_org_badtoken-3.snap similarity index 91% rename from tests/snapshots/cli__noseyparker_github_repos_list_org_badtoken-3.snap rename to tests/snapshots/github__github_repos_list_org_badtoken-3.snap index c4a495987..de8cdb044 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_org_badtoken-3.snap +++ b/tests/snapshots/github__github_repos_list_org_badtoken-3.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/github.rs expression: stderr --- Error: error making request: HTTP status client error (401 Unauthorized) for url (https://api.github.com/rate_limit) diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_noargs.snap b/tests/snapshots/github__github_repos_list_org_badtoken.snap similarity index 63% rename from tests/snapshots/cli__noseyparker_github_repos_list_noargs.snap rename to tests/snapshots/github__github_repos_list_org_badtoken.snap index 7749ce275..2ee2c85f0 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_noargs.snap +++ b/tests/snapshots/github__github_repos_list_org_badtoken.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/github.rs expression: status --- exit status: 2 diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_user_badtoken-2.snap b/tests/snapshots/github__github_repos_list_user_badtoken-2.snap similarity index 53% rename from tests/snapshots/cli__noseyparker_github_repos_list_user_badtoken-2.snap rename to tests/snapshots/github__github_repos_list_user_badtoken-2.snap index 02f5417a6..2caca4ffb 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_user_badtoken-2.snap +++ b/tests/snapshots/github__github_repos_list_user_badtoken-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/github.rs expression: stdout --- diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_user_badtoken-3.snap b/tests/snapshots/github__github_repos_list_user_badtoken-3.snap similarity index 91% rename from tests/snapshots/cli__noseyparker_github_repos_list_user_badtoken-3.snap rename to tests/snapshots/github__github_repos_list_user_badtoken-3.snap index c4a495987..de8cdb044 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_user_badtoken-3.snap +++ b/tests/snapshots/github__github_repos_list_user_badtoken-3.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/github.rs expression: stderr --- Error: error making request: HTTP status client error (401 Unauthorized) for url (https://api.github.com/rate_limit) diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_badtoken.snap b/tests/snapshots/github__github_repos_list_user_badtoken.snap similarity index 63% rename from tests/snapshots/cli__noseyparker_github_repos_list_badtoken.snap rename to tests/snapshots/github__github_repos_list_user_badtoken.snap index 7749ce275..2ee2c85f0 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_badtoken.snap +++ b/tests/snapshots/github__github_repos_list_user_badtoken.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/github.rs expression: status --- exit status: 2 diff --git a/tests/snapshots/cli__noseyparker_help-2.snap b/tests/snapshots/help__help-2.snap similarity index 98% rename from tests/snapshots/cli__noseyparker_help-2.snap rename to tests/snapshots/help__help-2.snap index d1684a371..9b2777150 100644 --- a/tests/snapshots/cli__noseyparker_help-2.snap +++ b/tests/snapshots/help__help-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stdout --- Find secrets and sensitive information in textual data diff --git a/tests/snapshots/cli__noseyparker_help-3.snap b/tests/snapshots/help__help-3.snap similarity index 56% rename from tests/snapshots/cli__noseyparker_help-3.snap rename to tests/snapshots/help__help-3.snap index 09fa0f492..30e5d1b37 100644 --- a/tests/snapshots/cli__noseyparker_help-3.snap +++ b/tests/snapshots/help__help-3.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stderr --- diff --git a/tests/snapshots/cli__noseyparker_help.snap b/tests/snapshots/help__help.snap similarity index 65% rename from tests/snapshots/cli__noseyparker_help.snap rename to tests/snapshots/help__help.snap index 66b7159c9..f892fabaa 100644 --- a/tests/snapshots/cli__noseyparker_help.snap +++ b/tests/snapshots/help__help.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: status --- exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_datastore-2.snap b/tests/snapshots/help__help_datastore-2.snap similarity index 97% rename from tests/snapshots/cli__noseyparker_help_datastore-2.snap rename to tests/snapshots/help__help_datastore-2.snap index e98b49bfc..438ef814b 100644 --- a/tests/snapshots/cli__noseyparker_help_datastore-2.snap +++ b/tests/snapshots/help__help_datastore-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stdout --- Manage datastores diff --git a/tests/snapshots/cli__noseyparker_help_datastore-3.snap b/tests/snapshots/help__help_datastore-3.snap similarity index 56% rename from tests/snapshots/cli__noseyparker_help_datastore-3.snap rename to tests/snapshots/help__help_datastore-3.snap index 09fa0f492..30e5d1b37 100644 --- a/tests/snapshots/cli__noseyparker_help_datastore-3.snap +++ b/tests/snapshots/help__help_datastore-3.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stderr --- diff --git a/tests/snapshots/cli__noseyparker_help_datastore.snap b/tests/snapshots/help__help_datastore.snap similarity index 65% rename from tests/snapshots/cli__noseyparker_help_datastore.snap rename to tests/snapshots/help__help_datastore.snap index 66b7159c9..f892fabaa 100644 --- a/tests/snapshots/cli__noseyparker_help_datastore.snap +++ b/tests/snapshots/help__help_datastore.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: status --- exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_github-2.snap b/tests/snapshots/help__help_github-2.snap similarity index 98% rename from tests/snapshots/cli__noseyparker_help_github-2.snap rename to tests/snapshots/help__help_github-2.snap index b28332e55..2586ed74f 100644 --- a/tests/snapshots/cli__noseyparker_help_github-2.snap +++ b/tests/snapshots/help__help_github-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stdout --- Interact with GitHub diff --git a/tests/snapshots/cli__noseyparker_help_github_repos-3.snap b/tests/snapshots/help__help_github-3.snap similarity index 56% rename from tests/snapshots/cli__noseyparker_help_github_repos-3.snap rename to tests/snapshots/help__help_github-3.snap index 09fa0f492..30e5d1b37 100644 --- a/tests/snapshots/cli__noseyparker_help_github_repos-3.snap +++ b/tests/snapshots/help__help_github-3.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stderr --- diff --git a/tests/snapshots/cli__noseyparker_help_github.snap b/tests/snapshots/help__help_github.snap similarity index 65% rename from tests/snapshots/cli__noseyparker_help_github.snap rename to tests/snapshots/help__help_github.snap index 66b7159c9..f892fabaa 100644 --- a/tests/snapshots/cli__noseyparker_help_github.snap +++ b/tests/snapshots/help__help_github.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: status --- exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_github_repos-2.snap b/tests/snapshots/help__help_github_repos-2.snap similarity index 98% rename from tests/snapshots/cli__noseyparker_help_github_repos-2.snap rename to tests/snapshots/help__help_github_repos-2.snap index 39709f0c2..cca983e67 100644 --- a/tests/snapshots/cli__noseyparker_help_github_repos-2.snap +++ b/tests/snapshots/help__help_github_repos-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stdout --- Interact with GitHub repositories diff --git a/tests/snapshots/cli__noseyparker_help_github-3.snap b/tests/snapshots/help__help_github_repos-3.snap similarity index 56% rename from tests/snapshots/cli__noseyparker_help_github-3.snap rename to tests/snapshots/help__help_github_repos-3.snap index 09fa0f492..30e5d1b37 100644 --- a/tests/snapshots/cli__noseyparker_help_github-3.snap +++ b/tests/snapshots/help__help_github_repos-3.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stderr --- diff --git a/tests/snapshots/cli__noseyparker_help_github_repos.snap b/tests/snapshots/help__help_github_repos.snap similarity index 65% rename from tests/snapshots/cli__noseyparker_help_github_repos.snap rename to tests/snapshots/help__help_github_repos.snap index 66b7159c9..f892fabaa 100644 --- a/tests/snapshots/cli__noseyparker_help_github_repos.snap +++ b/tests/snapshots/help__help_github_repos.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: status --- exit status: 0 diff --git a/tests/snapshots/help__help_github_repos_short-2.snap b/tests/snapshots/help__help_github_repos_short-2.snap new file mode 100644 index 000000000..b56a910e3 --- /dev/null +++ b/tests/snapshots/help__help_github_repos_short-2.snap @@ -0,0 +1,22 @@ +--- +source: tests/help.rs +expression: stdout +--- +Interact with GitHub repositories + +Usage: noseyparker github repos [OPTIONS] + +Commands: + list List repositories belonging to a specific user or organization + help Print this message or the help of the given subcommand(s) + +Options: + -h, --help Print help (see more with '--help') + +Global Options: + -v, --verbose... Enable verbose output + --color Enable or disable colored output [default: auto] [possible values: auto, + never, always] + --progress Enable or disable progress bars [default: auto] [possible values: auto, + never, always] + diff --git a/tests/snapshots/help__help_github_repos_short-3.snap b/tests/snapshots/help__help_github_repos_short-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_github_repos_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_github_repos_short.snap b/tests/snapshots/help__help_github_repos_short.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_github_repos_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_github_short-2.snap b/tests/snapshots/help__help_github_short-2.snap new file mode 100644 index 000000000..c383d0793 --- /dev/null +++ b/tests/snapshots/help__help_github_short-2.snap @@ -0,0 +1,22 @@ +--- +source: tests/help.rs +expression: stdout +--- +Interact with GitHub + +Usage: noseyparker github [OPTIONS] + +Commands: + repos Interact with GitHub repositories + help Print this message or the help of the given subcommand(s) + +Options: + -h, --help Print help (see more with '--help') + +Global Options: + -v, --verbose... Enable verbose output + --color Enable or disable colored output [default: auto] [possible values: auto, + never, always] + --progress Enable or disable progress bars [default: auto] [possible values: auto, + never, always] + diff --git a/tests/snapshots/help__help_github_short-3.snap b/tests/snapshots/help__help_github_short-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_github_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_github_short.snap b/tests/snapshots/help__help_github_short.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_github_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_report-2.snap b/tests/snapshots/help__help_report-2.snap similarity index 98% rename from tests/snapshots/cli__noseyparker_help_report-2.snap rename to tests/snapshots/help__help_report-2.snap index 6a1d2ce92..0487a13ac 100644 --- a/tests/snapshots/cli__noseyparker_help_report-2.snap +++ b/tests/snapshots/help__help_report-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stdout --- Report detailed scan findings diff --git a/tests/snapshots/help__help_report-3.snap b/tests/snapshots/help__help_report-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_report-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_report.snap b/tests/snapshots/help__help_report.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_report.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_report_short-2.snap b/tests/snapshots/help__help_report_short-2.snap new file mode 100644 index 000000000..13d4d227a --- /dev/null +++ b/tests/snapshots/help__help_report_short-2.snap @@ -0,0 +1,24 @@ +--- +source: tests/help.rs +expression: stdout +--- +Report detailed scan findings + +Usage: noseyparker report [OPTIONS] --datastore + +Options: + -d, --datastore Use the specified datastore path [env: NP_DATASTORE=] + -h, --help Print help (see more with '--help') + +Output Options: + -o, --output Write output to the specified path + -f, --format Write output in the specified format [default: human] [possible values: + human, json, jsonl] + +Global Options: + -v, --verbose... Enable verbose output + --color Enable or disable colored output [default: auto] [possible values: auto, + never, always] + --progress Enable or disable progress bars [default: auto] [possible values: auto, + never, always] + diff --git a/tests/snapshots/help__help_report_short-3.snap b/tests/snapshots/help__help_report_short-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_report_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_report_short.snap b/tests/snapshots/help__help_report_short.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_report_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_rules-2.snap b/tests/snapshots/help__help_rules-2.snap similarity index 97% rename from tests/snapshots/cli__noseyparker_help_rules-2.snap rename to tests/snapshots/help__help_rules-2.snap index 0a8abb058..7ccb741ed 100644 --- a/tests/snapshots/cli__noseyparker_help_rules-2.snap +++ b/tests/snapshots/help__help_rules-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stdout --- Manage rules diff --git a/tests/snapshots/help__help_rules-3.snap b/tests/snapshots/help__help_rules-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_rules-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_rules.snap b/tests/snapshots/help__help_rules.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_rules.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_scan-2.snap b/tests/snapshots/help__help_scan-2.snap similarity index 99% rename from tests/snapshots/cli__noseyparker_help_scan-2.snap rename to tests/snapshots/help__help_scan-2.snap index 17dbee429..1f93258bd 100644 --- a/tests/snapshots/cli__noseyparker_help_scan-2.snap +++ b/tests/snapshots/help__help_scan-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stdout --- Scan content for secrets diff --git a/tests/snapshots/help__help_scan-3.snap b/tests/snapshots/help__help_scan-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_scan-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_scan.snap b/tests/snapshots/help__help_scan.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_scan.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_scan_short-2.snap b/tests/snapshots/help__help_scan_short-2.snap new file mode 100644 index 000000000..22bb0ec53 --- /dev/null +++ b/tests/snapshots/help__help_scan_short-2.snap @@ -0,0 +1,32 @@ +--- +source: tests/help.rs +expression: stdout +--- +Scan content for secrets + +Usage: noseyparker scan [OPTIONS] --datastore [INPUT]... + +Options: + -d, --datastore Use the specified datastore path [env: NP_DATASTORE=] + -j, --jobs The number of parallel scanning jobs [default: DEFAULT] + -r, --rules Path of custom rules to use + -h, --help Print help (see more with '--help') + +Input Specifier Options: + [INPUT]... Path to a file, directory, or local Git repository to scan + --git-repo URL of a Git repository to clone and scan + --github-organization Name of a GitHub organization to enumerate and scan [aliases: + github-org] + --github-user Name of a GitHub user to enumerate and scan + +Content Discovery Options: + --max-file-size Do not scan files larger than the specified size [default: 100] + -i, --ignore Path of a custom ignore rules file to use + +Global Options: + -v, --verbose... Enable verbose output + --color Enable or disable colored output [default: auto] [possible values: auto, + never, always] + --progress Enable or disable progress bars [default: auto] [possible values: auto, + never, always] + diff --git a/tests/snapshots/help__help_scan_short-3.snap b/tests/snapshots/help__help_scan_short-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_scan_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_scan_short.snap b/tests/snapshots/help__help_scan_short.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_scan_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_short-2.snap b/tests/snapshots/help__help_short-2.snap new file mode 100644 index 000000000..d122462ac --- /dev/null +++ b/tests/snapshots/help__help_short-2.snap @@ -0,0 +1,28 @@ +--- +source: tests/help.rs +expression: stdout +--- +Find secrets and sensitive information in textual data + +Usage: noseyparker [OPTIONS] + +Commands: + scan Scan content for secrets + summarize Summarize scan findings + report Report detailed scan findings + github Interact with GitHub + datastore Manage datastores + rules Manage rules + help Print this message or the help of the given subcommand(s) + +Options: + -h, --help Print help (see more with '--help') + -V, --version Print version + +Global Options: + -v, --verbose... Enable verbose output + --color Enable or disable colored output [default: auto] [possible values: auto, + never, always] + --progress Enable or disable progress bars [default: auto] [possible values: auto, + never, always] + diff --git a/tests/snapshots/help__help_short-3.snap b/tests/snapshots/help__help_short-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_short.snap b/tests/snapshots/help__help_short.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_help_summarize-2.snap b/tests/snapshots/help__help_summarize-2.snap similarity index 98% rename from tests/snapshots/cli__noseyparker_help_summarize-2.snap rename to tests/snapshots/help__help_summarize-2.snap index 01181e403..05c65021b 100644 --- a/tests/snapshots/cli__noseyparker_help_summarize-2.snap +++ b/tests/snapshots/help__help_summarize-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stdout --- Summarize scan findings diff --git a/tests/snapshots/help__help_summarize-3.snap b/tests/snapshots/help__help_summarize-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_summarize-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_summarize.snap b/tests/snapshots/help__help_summarize.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_summarize.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_summarize_short-2.snap b/tests/snapshots/help__help_summarize_short-2.snap new file mode 100644 index 000000000..1f4192736 --- /dev/null +++ b/tests/snapshots/help__help_summarize_short-2.snap @@ -0,0 +1,24 @@ +--- +source: tests/help.rs +expression: stdout +--- +Summarize scan findings + +Usage: noseyparker summarize [OPTIONS] --datastore + +Options: + -d, --datastore Use the specified datastore path [env: NP_DATASTORE=] + -h, --help Print help (see more with '--help') + +Output Options: + -o, --output Write output to the specified path + -f, --format Write output in the specified format [default: human] [possible values: + human, json, jsonl] + +Global Options: + -v, --verbose... Enable verbose output + --color Enable or disable colored output [default: auto] [possible values: auto, + never, always] + --progress Enable or disable progress bars [default: auto] [possible values: auto, + never, always] + diff --git a/tests/snapshots/help__help_summarize_short-3.snap b/tests/snapshots/help__help_summarize_short-3.snap new file mode 100644 index 000000000..30e5d1b37 --- /dev/null +++ b/tests/snapshots/help__help_summarize_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_summarize_short.snap b/tests/snapshots/help__help_summarize_short.snap new file mode 100644 index 000000000..f892fabaa --- /dev/null +++ b/tests/snapshots/help__help_summarize_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_noargs-2.snap b/tests/snapshots/help__no_args-2.snap similarity index 56% rename from tests/snapshots/cli__noseyparker_github_repos_list_noargs-2.snap rename to tests/snapshots/help__no_args-2.snap index 02f5417a6..fb95b1a5b 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_noargs-2.snap +++ b/tests/snapshots/help__no_args-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stdout --- diff --git a/tests/snapshots/cli__noseyparker_no_args-3.snap b/tests/snapshots/help__no_args-3.snap similarity index 97% rename from tests/snapshots/cli__noseyparker_no_args-3.snap rename to tests/snapshots/help__no_args-3.snap index 716d9e29d..0f8f5edb1 100644 --- a/tests/snapshots/cli__noseyparker_no_args-3.snap +++ b/tests/snapshots/help__no_args-3.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: stderr --- Find secrets and sensitive information in textual data diff --git a/tests/snapshots/cli__noseyparker_github_repos_list_org_badtoken.snap b/tests/snapshots/help__no_args.snap similarity index 65% rename from tests/snapshots/cli__noseyparker_github_repos_list_org_badtoken.snap rename to tests/snapshots/help__no_args.snap index 7749ce275..9988d332e 100644 --- a/tests/snapshots/cli__noseyparker_github_repos_list_org_badtoken.snap +++ b/tests/snapshots/help__no_args.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/help.rs expression: status --- exit status: 2 diff --git a/tests/snapshots/cli__noseyparker_scan_secrets1-2.snap b/tests/snapshots/scan_workflow__scan_secrets1-2.snap similarity index 89% rename from tests/snapshots/cli__noseyparker_scan_secrets1-2.snap rename to tests/snapshots/scan_workflow__scan_secrets1-2.snap index 46ada641c..f65b8e2ff 100644 --- a/tests/snapshots/cli__noseyparker_scan_secrets1-2.snap +++ b/tests/snapshots/scan_workflow__scan_secrets1-2.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/scan_workflow.rs expression: stdout --- diff --git a/tests/snapshots/scan_workflow__scan_secrets1-3.snap b/tests/snapshots/scan_workflow__scan_secrets1-3.snap new file mode 100644 index 000000000..2ed81a9af --- /dev/null +++ b/tests/snapshots/scan_workflow__scan_secrets1-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/scan_workflow.rs +expression: stderr +--- + diff --git a/tests/snapshots/scan_workflow__scan_secrets1-4.snap b/tests/snapshots/scan_workflow__scan_secrets1-4.snap new file mode 100644 index 000000000..20c942991 --- /dev/null +++ b/tests/snapshots/scan_workflow__scan_secrets1-4.snap @@ -0,0 +1,5 @@ +--- +source: tests/scan_workflow.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/cli__noseyparker_scan_secrets1-5.snap b/tests/snapshots/scan_workflow__scan_secrets1-5.snap similarity index 89% rename from tests/snapshots/cli__noseyparker_scan_secrets1-5.snap rename to tests/snapshots/scan_workflow__scan_secrets1-5.snap index 4b3cfe8e4..ff414d432 100644 --- a/tests/snapshots/cli__noseyparker_scan_secrets1-5.snap +++ b/tests/snapshots/scan_workflow__scan_secrets1-5.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/scan_workflow.rs expression: stdout --- Finding 1/1: AWS API Key diff --git a/tests/snapshots/scan_workflow__scan_secrets1-6.snap b/tests/snapshots/scan_workflow__scan_secrets1-6.snap new file mode 100644 index 000000000..2ed81a9af --- /dev/null +++ b/tests/snapshots/scan_workflow__scan_secrets1-6.snap @@ -0,0 +1,5 @@ +--- +source: tests/scan_workflow.rs +expression: stderr +--- + diff --git a/tests/snapshots/cli__noseyparker_scan_secrets1-7.snap b/tests/snapshots/scan_workflow__scan_secrets1-7.snap similarity index 96% rename from tests/snapshots/cli__noseyparker_scan_secrets1-7.snap rename to tests/snapshots/scan_workflow__scan_secrets1-7.snap index be9cd1997..6dc6adac3 100644 --- a/tests/snapshots/cli__noseyparker_scan_secrets1-7.snap +++ b/tests/snapshots/scan_workflow__scan_secrets1-7.snap @@ -1,5 +1,5 @@ --- -source: tests/cli.rs +source: tests/scan_workflow.rs expression: json_output --- [ diff --git a/tests/snapshots/scan_workflow__scan_secrets1.snap b/tests/snapshots/scan_workflow__scan_secrets1.snap new file mode 100644 index 000000000..20c942991 --- /dev/null +++ b/tests/snapshots/scan_workflow__scan_secrets1.snap @@ -0,0 +1,5 @@ +--- +source: tests/scan_workflow.rs +expression: status +--- +exit status: 0 From a76f972a21547f2bd2735f95029c585635a964fc Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 16:08:18 -0500 Subject: [PATCH 07/15] Add more github enumeration tests --- tests/common/mod.rs | 1 + tests/github.rs | 43 +++++++++++++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 22671c7d0..6861079c2 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -10,6 +10,7 @@ pub use assert_fs::prelude::*; pub use assert_fs::{fixture::ChildPath, TempDir}; pub use insta::{assert_display_snapshot, assert_json_snapshot, assert_snapshot, with_settings}; pub use predicates::str::RegexPredicate; +pub use pretty_assertions::{assert_eq, assert_ne}; pub use std::path::Path; pub use std::process::Command; diff --git a/tests/github.rs b/tests/github.rs index 1a3ece56b..6fca02fbf 100644 --- a/tests/github.rs +++ b/tests/github.rs @@ -1,5 +1,7 @@ //! Tests for Nosey Parker `github` command +pub use pretty_assertions::{assert_eq, assert_ne}; + mod common; use common::*; @@ -28,7 +30,6 @@ fn github_repos_list_user_badtoken() { assert_cmd_snapshot!(cmd); } - // XXX Note: `octocat` is not a user under our control; it's a kind of test account owned by GitHub. // We are assuming that the `octocat` user's list of repositories will always include `Spoon-Knife`. @@ -49,8 +50,7 @@ fn handle_github_token(cmd: &mut Command) { fn github_repos_list_user_human_format() { let mut cmd = noseyparker!("github", "repos", "list", "--user", "octocat"); handle_github_token(&mut cmd); - cmd - .assert() + cmd.assert() .success() .stdout(predicates::str::contains("https://github.com/octocat/Spoon-Knife.git")) .stderr(predicates::str::is_empty()); @@ -60,24 +60,43 @@ fn github_repos_list_user_human_format() { fn github_repos_list_user_jsonl_format() { let mut cmd = noseyparker!("github", "repos", "list", "--user", "octocat", "--format", "jsonl"); handle_github_token(&mut cmd); - cmd - .assert() + cmd.assert() .success() .stdout(predicates::str::contains("\"https://github.com/octocat/Spoon-Knife.git\"\n")) .stderr(predicates::str::is_empty()); } #[test] -fn github_repos_list_user_json_format() { - let mut cmd = noseyparker!("github", "repos", "list", "--user", "octocat", "--format", "json"); +fn github_repos_list_multiple_user_dedupe_jsonl_format() { + let mut cmd = noseyparker!( + "github", "repos", "list", "--user", "octocat", "--user", "octocat", "--format", "jsonl" + ); handle_github_token(&mut cmd); - let cmd = cmd - .assert() + let cmd = cmd.assert() .success() + .stdout(predicates::str::contains("\"https://github.com/octocat/Spoon-Knife.git\"\n")) .stderr(predicates::str::is_empty()); + // Ensure that output is sorted and there are no dupes + let stdout = String::from_utf8(cmd.get_output().stdout.clone()).expect("noseyparker output should be utf-8"); + let stdout_lines: Vec = stdout.lines().map(|s| s.to_string()).collect(); + let mut sorted_stdout_lines = stdout_lines.clone(); + sorted_stdout_lines.sort(); + sorted_stdout_lines.dedup(); + assert_eq!(stdout_lines, sorted_stdout_lines); +} + +#[test] +fn github_repos_list_user_json_format() { + let mut cmd = noseyparker!("github", "repos", "list", "--user", "octocat", "--format", "json"); + handle_github_token(&mut cmd); + let cmd = cmd.assert().success().stderr(predicates::str::is_empty()); + let output = &cmd.get_output().stdout; - let json_parsed: Vec = serde_json::from_slice(output).expect("output should be well-formed JSON"); - assert!(json_parsed.contains(&String::from("https://github.com/octocat/Spoon-Knife.git")), - "JSON output does not contain https://github.com/octocat/Spoon-Knife.git: {json_parsed:?}"); + let json_parsed: Vec = + serde_json::from_slice(output).expect("output should be well-formed JSON"); + assert!( + json_parsed.contains(&String::from("https://github.com/octocat/Spoon-Knife.git")), + "JSON output does not contain https://github.com/octocat/Spoon-Knife.git: {json_parsed:?}" + ); } From 308d1c17eeccdfef6b6d162f30611cc657a40d09 Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 16:10:04 -0500 Subject: [PATCH 08/15] Fix GitHub Actions --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7888a99cd..4fc2087c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,10 +143,11 @@ jobs: - name: Run tests env: - # We use the GitHub Actions automatic token when running tests, to avoid - # spurious failures from rate limiting when testing Nosey Parker's github - # enumeration capabilities. - - NP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # We use the GitHub Actions automatic token when running tests, to avoid + # spurious failures from rate limiting when testing Nosey Parker's github + # enumeration capabilities. + NP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: cargo test --release --verbose --locked - name: Check rules From 24712912d4b34f1f251df39b437d434d5f75a313 Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 16:15:27 -0500 Subject: [PATCH 09/15] Fix GitHub Actions --- .github/workflows/ci.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fc2087c5..275f2c305 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,13 +71,18 @@ jobs: key: ${{ runner.os }}-${{ steps.install-rust-toolchain.cachekey }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('**/Cargo.lock') }}-tests - name: Build - run: cargo build --verbose --locked + run: cargo build --locked --verbose - name: Run tests - run: cargo test --verbose --locked + env: + # We use the GitHub Actions automatic token when running tests, to avoid + # spurious failures from rate limiting when testing Nosey Parker's github + # enumeration capabilities. + NP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: cargo test --locked --release --verbose - name: Check rules - run: cargo run -- rules check data/default/rules --warnings-as-errors + run: cargo run --locked -- rules check data/default/rules --warnings-as-errors docs: name: Docs @@ -109,7 +114,7 @@ jobs: - name: Check documentation env: RUSTDOCFLAGS: -D warnings - run: cargo doc --no-deps --document-private-items + run: cargo doc --locked --no-deps --document-private-items release_build: name: Release Build @@ -139,7 +144,7 @@ jobs: key: ${{ runner.os }}-${{ steps.install-rust-toolchain.cachekey }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('**/Cargo.lock') }}-release_build - name: Build - run: cargo build --release --verbose --locked + run: cargo build --locked --release --verbose - name: Run tests env: @@ -147,11 +152,10 @@ jobs: # spurious failures from rate limiting when testing Nosey Parker's github # enumeration capabilities. NP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: cargo test --release --verbose --locked + run: cargo test --locked --release --verbose - name: Check rules - run: cargo run --release -- rules check data/default/rules --warnings-as-errors + run: cargo run --locked --release -- rules check data/default/rules --warnings-as-errors - name: Upload release binary uses: actions/upload-artifact@v3 From d94144422e72ab7c4561be7fa10b31617393d046 Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Wed, 15 Feb 2023 16:43:03 -0500 Subject: [PATCH 10/15] Rename GitHub Actions workflows for clarity --- .github/workflows/ci.yml | 2 +- .github/workflows/docker-image.yml | 1 + .github/workflows/rust-clippy.yml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 275f2c305..b9f322902 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: run: cargo doc --locked --no-deps --document-private-items release_build: - name: Release Build + name: Tests (release build; ubuntu-20.04.stable) runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index ed8bb4463..965847758 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -9,6 +9,7 @@ on: jobs: build: + name: Build, Test, and (Optionally) Publish runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/rust-clippy.yml b/.github/workflows/rust-clippy.yml index 90584c820..39eef3e32 100644 --- a/.github/workflows/rust-clippy.yml +++ b/.github/workflows/rust-clippy.yml @@ -7,7 +7,7 @@ # More details at https://github.com/rust-lang/rust-clippy # and https://rust-lang.github.io/rust-clippy/ -name: rust-clippy analyze +name: Clippy on: push: @@ -20,7 +20,7 @@ on: jobs: rust-clippy-analyze: - name: Run rust-clippy analyzing + name: Clippy Analysis runs-on: ubuntu-latest permissions: contents: read From 2f793edb44993d08b5b53a064c622e19b5b2e69e Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Thu, 16 Feb 2023 17:30:02 -0500 Subject: [PATCH 11/15] Checkpoint --- Cargo.lock | 21 ++ Cargo.toml | 2 +- src/bin/noseyparker/args.rs | 9 +- src/bin/noseyparker/cmd_scan.rs | 68 +++++- src/datastore.rs | 4 +- src/git_binary.rs | 30 ++- src/git_url.rs | 173 +++++++++++++ src/github.rs | 2 +- src/input_enumerator.rs | 4 - src/lib.rs | 1 + tests/common/mod.rs | 9 +- tests/scan.rs | 117 --------- tests/scan_workflow.rs | 30 --- .../github__github_repos_list_noargs-2.snap | 5 - ...hub__github_repos_list_org_badtoken-2.snap | 5 - ...ub__github_repos_list_user_badtoken-2.snap | 5 - tests/snapshots/help__help-3.snap | 5 - tests/snapshots/help__help_datastore-3.snap | 5 - tests/snapshots/help__help_github-3.snap | 5 - .../snapshots/help__help_github_repos-3.snap | 5 - .../help__help_github_repos_short-3.snap | 5 - .../help__help_github_repos_short.snap | 5 - .../snapshots/help__help_github_short-3.snap | 5 - tests/snapshots/help__help_github_short.snap | 5 - tests/snapshots/help__help_report-3.snap | 5 - tests/snapshots/help__help_report.snap | 5 - .../snapshots/help__help_report_short-3.snap | 5 - tests/snapshots/help__help_report_short.snap | 5 - tests/snapshots/help__help_rules-3.snap | 5 - tests/snapshots/help__help_rules.snap | 5 - tests/snapshots/help__help_scan-3.snap | 5 - tests/snapshots/help__help_scan.snap | 5 - tests/snapshots/help__help_scan_short-3.snap | 5 - tests/snapshots/help__help_scan_short.snap | 5 - tests/snapshots/help__help_short-3.snap | 5 - tests/snapshots/help__help_short.snap | 5 - tests/snapshots/help__help_summarize-3.snap | 5 - tests/snapshots/help__help_summarize.snap | 5 - .../help__help_summarize_short-3.snap | 5 - .../snapshots/help__help_summarize_short.snap | 5 - tests/snapshots/help__no_args-2.snap | 5 - .../scan_workflow__scan_secrets1-3.snap | 5 - .../scan_workflow__scan_secrets1-4.snap | 5 - .../scan_workflow__scan_secrets1-6.snap | 5 - .../scan_workflow__scan_secrets1.snap | 5 - ...er_github__github_repos_list_noargs-2.snap | 5 + ...r_github__github_repos_list_noargs-3.snap} | 2 +- ...rker_github__github_repos_list_noargs.snap | 5 + ...hub__github_repos_list_org_badtoken-2.snap | 5 + ...ub__github_repos_list_org_badtoken-3.snap} | 2 +- ...ithub__github_repos_list_org_badtoken.snap | 5 + ...ub__github_repos_list_user_badtoken-2.snap | 5 + ...b__github_repos_list_user_badtoken-3.snap} | 2 +- ...thub__github_repos_list_user_badtoken.snap | 5 + ...nap => test_noseyparker_help__help-2.snap} | 2 +- .../test_noseyparker_help__help-3.snap | 5 + ....snap => test_noseyparker_help__help.snap} | 2 +- ...t_noseyparker_help__help_datastore-2.snap} | 2 +- ...st_noseyparker_help__help_datastore-3.snap | 5 + ...est_noseyparker_help__help_datastore.snap} | 2 +- ...test_noseyparker_help__help_github-2.snap} | 2 +- .../test_noseyparker_help__help_github-3.snap | 5 + ...> test_noseyparker_help__help_github.snap} | 2 +- ...oseyparker_help__help_github_repos-2.snap} | 2 +- ...noseyparker_help__help_github_repos-3.snap | 5 + ..._noseyparker_help__help_github_repos.snap} | 2 +- ...rker_help__help_github_repos_short-2.snap} | 2 +- ...arker_help__help_github_repos_short-3.snap | 5 + ...yparker_help__help_github_repos_short.snap | 5 + ...oseyparker_help__help_github_short-2.snap} | 2 +- ...noseyparker_help__help_github_short-3.snap | 5 + ...t_noseyparker_help__help_github_short.snap | 5 + ...test_noseyparker_help__help_report-2.snap} | 2 +- .../test_noseyparker_help__help_report-3.snap | 5 + .../test_noseyparker_help__help_report.snap | 5 + ...oseyparker_help__help_report_short-2.snap} | 2 +- ...noseyparker_help__help_report_short-3.snap | 5 + ...t_noseyparker_help__help_report_short.snap | 5 + ... test_noseyparker_help__help_rules-2.snap} | 2 +- .../test_noseyparker_help__help_rules-3.snap | 5 + .../test_noseyparker_help__help_rules.snap | 5 + ...> test_noseyparker_help__help_scan-2.snap} | 7 +- .../test_noseyparker_help__help_scan-3.snap | 5 + .../test_noseyparker_help__help_scan.snap | 5 + ..._noseyparker_help__help_scan_short-2.snap} | 4 +- ...t_noseyparker_help__help_scan_short-3.snap | 5 + ...est_noseyparker_help__help_scan_short.snap | 5 + ... test_noseyparker_help__help_short-2.snap} | 2 +- .../test_noseyparker_help__help_short-3.snap | 5 + .../test_noseyparker_help__help_short.snap | 5 + ...t_noseyparker_help__help_summarize-2.snap} | 2 +- ...st_noseyparker_help__help_summarize-3.snap | 5 + ...test_noseyparker_help__help_summarize.snap | 5 + ...yparker_help__help_summarize_short-2.snap} | 2 +- ...eyparker_help__help_summarize_short-3.snap | 5 + ...oseyparker_help__help_summarize_short.snap | 5 + .../test_noseyparker_help__no_args-2.snap | 5 + ... => test_noseyparker_help__no_args-3.snap} | 2 +- ...ap => test_noseyparker_help__no_args.snap} | 2 +- ...st_noseyparker_scan__scan_secrets1-2.snap} | 2 +- ...est_noseyparker_scan__scan_secrets1-3.snap | 5 + ...est_noseyparker_scan__scan_secrets1-4.snap | 5 + ...st_noseyparker_scan__scan_secrets1-5.snap} | 2 +- ...est_noseyparker_scan__scan_secrets1-6.snap | 5 + ...st_noseyparker_scan__scan_secrets1-7.snap} | 2 +- .../test_noseyparker_scan__scan_secrets1.snap | 5 + ...can__test_scan_git_url__file_scheme-2.snap | 5 + ...can__test_scan_git_url__file_scheme-3.snap | 8 + ...scan__test_scan_git_url__file_scheme.snap} | 2 +- ...can__test_scan_git_url__http_scheme-2.snap | 5 + ...can__test_scan_git_url__http_scheme-3.snap | 8 + ...scan__test_scan_git_url__http_scheme.snap} | 2 +- ..._scan__test_scan_git_url__no_scheme-2.snap | 5 + ..._scan__test_scan_git_url__no_scheme-3.snap | 8 + ...r_scan__test_scan_git_url__no_scheme.snap} | 2 +- ...scan__test_scan_git_url__ssh_scheme-2.snap | 5 + ...scan__test_scan_git_url__ssh_scheme-3.snap | 8 + ...r_scan__test_scan_git_url__ssh_scheme.snap | 5 + .../{github.rs => test_noseyparker_github.rs} | 6 +- tests/{help.rs => test_noseyparker_help.rs} | 0 tests/test_noseyparker_scan.rs | 228 ++++++++++++++++++ 121 files changed, 792 insertions(+), 369 deletions(-) create mode 100644 src/git_url.rs delete mode 100644 tests/scan.rs delete mode 100644 tests/scan_workflow.rs delete mode 100644 tests/snapshots/github__github_repos_list_noargs-2.snap delete mode 100644 tests/snapshots/github__github_repos_list_org_badtoken-2.snap delete mode 100644 tests/snapshots/github__github_repos_list_user_badtoken-2.snap delete mode 100644 tests/snapshots/help__help-3.snap delete mode 100644 tests/snapshots/help__help_datastore-3.snap delete mode 100644 tests/snapshots/help__help_github-3.snap delete mode 100644 tests/snapshots/help__help_github_repos-3.snap delete mode 100644 tests/snapshots/help__help_github_repos_short-3.snap delete mode 100644 tests/snapshots/help__help_github_repos_short.snap delete mode 100644 tests/snapshots/help__help_github_short-3.snap delete mode 100644 tests/snapshots/help__help_github_short.snap delete mode 100644 tests/snapshots/help__help_report-3.snap delete mode 100644 tests/snapshots/help__help_report.snap delete mode 100644 tests/snapshots/help__help_report_short-3.snap delete mode 100644 tests/snapshots/help__help_report_short.snap delete mode 100644 tests/snapshots/help__help_rules-3.snap delete mode 100644 tests/snapshots/help__help_rules.snap delete mode 100644 tests/snapshots/help__help_scan-3.snap delete mode 100644 tests/snapshots/help__help_scan.snap delete mode 100644 tests/snapshots/help__help_scan_short-3.snap delete mode 100644 tests/snapshots/help__help_scan_short.snap delete mode 100644 tests/snapshots/help__help_short-3.snap delete mode 100644 tests/snapshots/help__help_short.snap delete mode 100644 tests/snapshots/help__help_summarize-3.snap delete mode 100644 tests/snapshots/help__help_summarize.snap delete mode 100644 tests/snapshots/help__help_summarize_short-3.snap delete mode 100644 tests/snapshots/help__help_summarize_short.snap delete mode 100644 tests/snapshots/help__no_args-2.snap delete mode 100644 tests/snapshots/scan_workflow__scan_secrets1-3.snap delete mode 100644 tests/snapshots/scan_workflow__scan_secrets1-4.snap delete mode 100644 tests/snapshots/scan_workflow__scan_secrets1-6.snap delete mode 100644 tests/snapshots/scan_workflow__scan_secrets1.snap create mode 100644 tests/snapshots/test_noseyparker_github__github_repos_list_noargs-2.snap rename tests/snapshots/{github__github_repos_list_noargs-3.snap => test_noseyparker_github__github_repos_list_noargs-3.snap} (59%) create mode 100644 tests/snapshots/test_noseyparker_github__github_repos_list_noargs.snap create mode 100644 tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken-2.snap rename tests/snapshots/{github__github_repos_list_org_badtoken-3.snap => test_noseyparker_github__github_repos_list_org_badtoken-3.snap} (85%) create mode 100644 tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken.snap create mode 100644 tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken-2.snap rename tests/snapshots/{github__github_repos_list_user_badtoken-3.snap => test_noseyparker_github__github_repos_list_user_badtoken-3.snap} (85%) create mode 100644 tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken.snap rename tests/snapshots/{help__help-2.snap => test_noseyparker_help__help-2.snap} (97%) create mode 100644 tests/snapshots/test_noseyparker_help__help-3.snap rename tests/snapshots/{help__help_datastore.snap => test_noseyparker_help__help.snap} (51%) rename tests/snapshots/{help__help_datastore-2.snap => test_noseyparker_help__help_datastore-2.snap} (96%) create mode 100644 tests/snapshots/test_noseyparker_help__help_datastore-3.snap rename tests/snapshots/{help__help_github.snap => test_noseyparker_help__help_datastore.snap} (51%) rename tests/snapshots/{help__help_github-2.snap => test_noseyparker_help__help_github-2.snap} (97%) create mode 100644 tests/snapshots/test_noseyparker_help__help_github-3.snap rename tests/snapshots/{help__help_github_repos.snap => test_noseyparker_help__help_github.snap} (51%) rename tests/snapshots/{help__help_github_repos-2.snap => test_noseyparker_help__help_github_repos-2.snap} (96%) create mode 100644 tests/snapshots/test_noseyparker_help__help_github_repos-3.snap rename tests/snapshots/{help__help.snap => test_noseyparker_help__help_github_repos.snap} (51%) rename tests/snapshots/{help__help_github_repos_short-2.snap => test_noseyparker_help__help_github_repos_short-2.snap} (94%) create mode 100644 tests/snapshots/test_noseyparker_help__help_github_repos_short-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_github_repos_short.snap rename tests/snapshots/{help__help_github_short-2.snap => test_noseyparker_help__help_github_short-2.snap} (94%) create mode 100644 tests/snapshots/test_noseyparker_help__help_github_short-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_github_short.snap rename tests/snapshots/{help__help_report-2.snap => test_noseyparker_help__help_report-2.snap} (97%) create mode 100644 tests/snapshots/test_noseyparker_help__help_report-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_report.snap rename tests/snapshots/{help__help_report_short-2.snap => test_noseyparker_help__help_report_short-2.snap} (95%) create mode 100644 tests/snapshots/test_noseyparker_help__help_report_short-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_report_short.snap rename tests/snapshots/{help__help_rules-2.snap => test_noseyparker_help__help_rules-2.snap} (96%) create mode 100644 tests/snapshots/test_noseyparker_help__help_rules-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_rules.snap rename tests/snapshots/{help__help_scan-2.snap => test_noseyparker_help__help_scan-2.snap} (95%) create mode 100644 tests/snapshots/test_noseyparker_help__help_scan-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_scan.snap rename tests/snapshots/{help__help_scan_short-2.snap => test_noseyparker_help__help_scan_short-2.snap} (92%) create mode 100644 tests/snapshots/test_noseyparker_help__help_scan_short-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_scan_short.snap rename tests/snapshots/{help__help_short-2.snap => test_noseyparker_help__help_short-2.snap} (95%) create mode 100644 tests/snapshots/test_noseyparker_help__help_short-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_short.snap rename tests/snapshots/{help__help_summarize-2.snap => test_noseyparker_help__help_summarize-2.snap} (97%) create mode 100644 tests/snapshots/test_noseyparker_help__help_summarize-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_summarize.snap rename tests/snapshots/{help__help_summarize_short-2.snap => test_noseyparker_help__help_summarize_short-2.snap} (95%) create mode 100644 tests/snapshots/test_noseyparker_help__help_summarize_short-3.snap create mode 100644 tests/snapshots/test_noseyparker_help__help_summarize_short.snap create mode 100644 tests/snapshots/test_noseyparker_help__no_args-2.snap rename tests/snapshots/{help__no_args-3.snap => test_noseyparker_help__no_args-3.snap} (95%) rename tests/snapshots/{github__github_repos_list_org_badtoken.snap => test_noseyparker_help__no_args.snap} (51%) rename tests/snapshots/{scan_workflow__scan_secrets1-2.snap => test_noseyparker_scan__scan_secrets1-2.snap} (87%) create mode 100644 tests/snapshots/test_noseyparker_scan__scan_secrets1-3.snap create mode 100644 tests/snapshots/test_noseyparker_scan__scan_secrets1-4.snap rename tests/snapshots/{scan_workflow__scan_secrets1-5.snap => test_noseyparker_scan__scan_secrets1-5.snap} (86%) create mode 100644 tests/snapshots/test_noseyparker_scan__scan_secrets1-6.snap rename tests/snapshots/{scan_workflow__scan_secrets1-7.snap => test_noseyparker_scan__scan_secrets1-7.snap} (96%) create mode 100644 tests/snapshots/test_noseyparker_scan__scan_secrets1.snap create mode 100644 tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme-2.snap create mode 100644 tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme-3.snap rename tests/snapshots/{github__github_repos_list_noargs.snap => test_noseyparker_scan__test_scan_git_url__file_scheme.snap} (51%) create mode 100644 tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme-2.snap create mode 100644 tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme-3.snap rename tests/snapshots/{github__github_repos_list_user_badtoken.snap => test_noseyparker_scan__test_scan_git_url__http_scheme.snap} (51%) create mode 100644 tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme-2.snap create mode 100644 tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme-3.snap rename tests/snapshots/{help__no_args.snap => test_noseyparker_scan__test_scan_git_url__no_scheme.snap} (51%) create mode 100644 tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme-2.snap create mode 100644 tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme-3.snap create mode 100644 tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme.snap rename tests/{github.rs => test_noseyparker_github.rs} (97%) rename tests/{help.rs => test_noseyparker_help.rs} (100%) create mode 100644 tests/test_noseyparker_scan.rs diff --git a/Cargo.lock b/Cargo.lock index d70c5c810..37de504b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,11 +80,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9834fcc22e0874394a010230586367d4a3e9f11b560f469262678547e1d2575e" dependencies = [ "bstr 1.2.0", + "concolor", "doc-comment", "predicates", "predicates-core", "predicates-tree", "wait-timeout", + "yansi", ] [[package]] @@ -401,6 +403,23 @@ dependencies = [ "ryu", ] +[[package]] +name = "concolor" +version = "0.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "318d6c16e73b3a900eb212ad6a82fc7d298c5ab8184c7a9998646455bc474a16" +dependencies = [ + "bitflags", + "concolor-query", + "is-terminal", +] + +[[package]] +name = "concolor-query" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82a90734b3d5dcf656e7624cca6bce9c3a90ee11f900e80141a7427ccfb3d317" + [[package]] name = "console" version = "0.15.5" @@ -2566,12 +2585,14 @@ version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" dependencies = [ + "concolor", "difflib", "float-cmp", "itertools", "normalize-line-endings", "predicates-core", "regex", + "yansi", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4b382097c..8b05172d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,7 @@ url = "2.3" walkdir = "2.3" [dev-dependencies] -assert_cmd = "2.0" +assert_cmd = { version = "2.0", features = ["color-auto"] } assert_fs = "1.0" criterion = "0.4" escargot = "0.5" diff --git a/src/bin/noseyparker/args.rs b/src/bin/noseyparker/args.rs index 4dd87dcc4..3f5e69d34 100644 --- a/src/bin/noseyparker/args.rs +++ b/src/bin/noseyparker/args.rs @@ -1,8 +1,9 @@ use anyhow::{Context, Result}; use clap::{crate_description, crate_version, ArgAction, Args, Parser, Subcommand, ValueEnum}; - use std::path::PathBuf; +use noseyparker::git_url::GitUrl; + // ----------------------------------------------------------------------------- // command-line args // ----------------------------------------------------------------------------- @@ -311,12 +312,14 @@ pub struct ScanArgs { #[command(next_help_heading = "Input Specifier Options")] pub struct ScanInputArgs { /// Path to a file, directory, or local Git repository to scan - #[arg(value_name="INPUT", required_unless_present_any(["github_user", "github_organization", "git_repo"]), display_order=1)] + #[arg(value_name="INPUT", required_unless_present_any(["github_user", "github_organization", "git_url"]), display_order=1)] pub path_inputs: Vec, /// URL of a Git repository to clone and scan + /// + /// Only https URLs without credentials, query parameters, or fragment identifiers are supported. #[arg(long, value_name = "URL", display_order = 10)] - pub git_repo: Vec, + pub git_url: Vec, /// Name of a GitHub user to enumerate and scan #[arg(long, value_name = "NAME", display_order = 20)] diff --git a/src/bin/noseyparker/cmd_scan.rs b/src/bin/noseyparker/cmd_scan.rs index 32a388271..eeb75c18d 100644 --- a/src/bin/noseyparker/cmd_scan.rs +++ b/src/bin/noseyparker/cmd_scan.rs @@ -1,7 +1,8 @@ -use anyhow::{Context, Result}; +use anyhow::{Context, Result, bail}; use git_repository as git; use indicatif::{HumanBytes, HumanCount, HumanDuration}; use rayon::prelude::*; +use std::str::FromStr; use std::sync::mpsc; use std::sync::Mutex; use std::time::Instant; @@ -15,6 +16,7 @@ use noseyparker::datastore::Datastore; use noseyparker::defaults::DEFAULT_IGNORE_RULES; use noseyparker::github; use noseyparker::git_binary::Git; +use noseyparker::git_url::GitUrl; use noseyparker::input_enumerator::{open_git_repo, FileResult, FilesystemEnumerator}; use noseyparker::location; use noseyparker::match_type::Match; @@ -73,9 +75,17 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> user: args.input_args.github_user.clone(), organization: args.input_args.github_organization.clone(), }; - let mut repo_urls = args.input_args.git_repo.clone(); + let mut repo_urls = args.input_args.git_url.clone(); if !repo_specifiers.is_empty() { - repo_urls.extend(github::enumerate_repo_urls(&repo_specifiers)?); + for repo_string in github::enumerate_repo_urls(&repo_specifiers)? { + match GitUrl::from_str(&repo_string) { + Ok(repo_url) => repo_urls.push(repo_url), + Err(e) => { + error!("Failed to parse repo URL from {repo_string}: {e}"); + continue; + } + } + } } repo_urls.sort(); repo_urls.dedup(); @@ -96,8 +106,15 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> let clones_dir = tmpdir.join("clones"); let git = Git::new(); + // FIXME: put a progress meter around this; disable Git's built-in progress for repo_url in repo_urls { - let output_dir = clones_dir.join(&repo_url); + let output_dir = match clone_destination(&clones_dir, &repo_url) { + Err(e) => { + error!("Failed to determine output directory for {repo_url}: {e}"); + continue + } + Ok(output_dir) => output_dir, + }; // First, try to update an existing clone, and if that fails, do a fresh clone if output_dir.is_dir() { @@ -121,10 +138,13 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> } } + if input_roots.is_empty() { + bail!("No inputs to scan"); + } + // --------------------------------------------------------------------------------------------- // Enumerate initial filesystem inputs // --------------------------------------------------------------------------------------------- - debug_assert!(!input_roots.is_empty()); let inputs = { let mut progress = Progress::new_bytes_spinner("Enumerating inputs...", progress_enabled); @@ -421,3 +441,41 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()> Ok(()) } + + + +/// Get a path for a local clone of the given git URL underneath `root`. +fn clone_destination(root: &std::path::Path, repo: &GitUrl) -> Result { + Ok(root.join(repo.to_path_buf())) +} + +#[cfg(test)] +mod test { + macro_rules! clone_destination_success_tests { + ($($case_name:ident: ($root:expr, $repo:expr) => $expected:expr,)*) => { + mod clone_destination { + use noseyparker::git_url::GitUrl; + use pretty_assertions::assert_eq; + use std::path::{PathBuf, Path}; + use std::str::FromStr; + use super::super::clone_destination; + + $( + #[test] + fn $case_name() { + let expected: Option = Some(Path::new($expected).to_owned()); + + let root = Path::new($root); + let repo = GitUrl::from_str($repo).expect("repo should be a URL"); + assert_eq!(clone_destination(root, &repo).ok(), expected); + } + )* + } + } + } + + clone_destination_success_tests! { + https_01: ("rel_root", "https://example.com/testrepo.git") => "rel_root/https/example.com/testrepo.git", + https_02: ("/abs_root", "https://example.com/testrepo.git") => "/abs_root/https/example.com/testrepo.git", + } +} diff --git a/src/datastore.rs b/src/datastore.rs index f2d8b51e3..e18fc1d2c 100644 --- a/src/datastore.rs +++ b/src/datastore.rs @@ -37,8 +37,10 @@ impl Datastore { let db_path = root_dir.join("datastore.db"); let conn = Self::new_connection(&db_path) .with_context(|| format!("Failed to open database at {}", db_path.display()))?; + let root_dir = root_dir.canonicalize() + .with_context(|| format!("Failed to canonicalize datastore path at {}", root_dir.display()))?; let mut ds = Self { - root_dir: root_dir.to_owned(), + root_dir, conn, }; ds.migrate() diff --git a/src/git_binary.rs b/src/git_binary.rs index 80b84c2d3..edd984362 100644 --- a/src/git_binary.rs +++ b/src/git_binary.rs @@ -2,9 +2,10 @@ use std::path::Path; use std::process::{Command, ExitStatus, Stdio}; use tracing::{debug, debug_span}; +use crate::git_url::GitUrl; + #[derive(Debug)] -pub enum GitError -{ +pub enum GitError { IOError(std::io::Error), GitError { stdout: Vec, @@ -22,12 +23,17 @@ impl From for GitError { impl std::fmt::Display for GitError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - GitError::IOError(e) => write!(f, "{e}"), + GitError::IOError(e) => write!(f, "git execution failed: {e}"), GitError::GitError { - stdout: _, - stderr: _, + stdout, + stderr, status, - } => write!(f, "git execution failed: {}", status), + } => write!( + f, + "git execution failed\ncode={status}\nstdout=```\n{}```\nstderr=```\n{}```", + String::from_utf8_lossy(stdout), + String::from_utf8_lossy(stderr) + ), } } } @@ -36,7 +42,7 @@ impl std::error::Error for GitError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { GitError::IOError(e) => Some(e), - GitError::GitError {..} => None, + GitError::GitError { .. } => None, } } } @@ -72,7 +78,7 @@ impl Git { cmd } - pub fn update_mirrored_clone(&self, repo_url: &str, output_dir: &Path) -> Result<(), GitError> { + pub fn update_mirrored_clone(&self, repo_url: &GitUrl, output_dir: &Path) -> Result<(), GitError> { let _span = debug_span!("git_update", "{repo_url} {}", output_dir.display()).entered(); debug!("Attempting to update clone of {repo_url} at {}", output_dir.display()); @@ -95,14 +101,18 @@ impl Git { Ok(()) } - pub fn create_fresh_mirrored_clone(&self, repo_url: &str, output_dir: &Path) -> Result<(), GitError> { + pub fn create_fresh_mirrored_clone( + &self, + repo_url: &GitUrl, + output_dir: &Path, + ) -> Result<(), GitError> { let _span = debug_span!("git_clone", "{repo_url} {}", output_dir.display()).entered(); debug!("Attempting to create fresh clone of {} at {}", repo_url, output_dir.display()); let mut cmd = self.git(); cmd.arg("clone") .arg("--mirror") - .arg(repo_url) + .arg(repo_url.as_str()) .arg(output_dir); debug!("{cmd:#?}"); diff --git a/src/git_url.rs b/src/git_url.rs new file mode 100644 index 000000000..3814dae06 --- /dev/null +++ b/src/git_url.rs @@ -0,0 +1,173 @@ +use std::path::PathBuf; +use url::Url; + +#[derive(Clone, PartialEq, Eq, Debug, PartialOrd, Ord)] +pub struct GitUrl(Url); + +impl GitUrl { + /// Convert this URL into a path. + /// This avoids potential path traversal issues with URLs like + /// `https://example.com/../boom.git`. + pub fn to_path_buf(&self) -> std::path::PathBuf { + let mut result = PathBuf::new(); + result.push(self.0.scheme()); + + let host_string = match self.0.host().expect("host should be non-empty") { + url::Host::Domain(host) => host.to_owned(), + url::Host::Ipv4(addr) => addr.to_string(), + url::Host::Ipv6(addr) => addr.to_string(), + }; + if let Some(port) = self.0.port() { + result.push(format!("{host_string}:{port}")); + } else { + result.push(host_string); + } + result.extend(self.0.path_segments().expect("path segments should decode")); + + result + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl std::fmt::Display for GitUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0.as_str()) + } +} + +const GIT_URL_ERROR_MESSAGE: &str = + "only https URLs without credentials, query parameters, or fragment identifiers are supported"; + +impl std::str::FromStr for GitUrl { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match Url::parse(s) { + Err(_e) => Err(GIT_URL_ERROR_MESSAGE), + Ok(url) => Self::try_from(url), + } + } +} + +impl TryFrom for GitUrl { + type Error = &'static str; + + fn try_from(url: Url) -> Result { + if url.scheme() != "https" { + return Err(GIT_URL_ERROR_MESSAGE); + } + + if url.host().is_none() { + return Err(GIT_URL_ERROR_MESSAGE); + } + + if !url.username().is_empty() || url.password().is_some() { + return Err(GIT_URL_ERROR_MESSAGE); + } + + if url.query().is_some() { + return Err(GIT_URL_ERROR_MESSAGE); + } + + if url.fragment().is_some() { + return Err(GIT_URL_ERROR_MESSAGE); + } + + match url.path_segments() { + None => return Err(GIT_URL_ERROR_MESSAGE), + Some(segments) => { + for segment in segments { + if segment == ".." { + return Err(GIT_URL_ERROR_MESSAGE); + } + } + } + } + + Ok(GitUrl(url)) + } +} + +#[cfg(test)] +mod test { + use super::*; + use pretty_assertions::assert_eq; + use std::path::Path; + use std::str::FromStr; + + #[test] + fn bad_scheme_01() { + assert!(GitUrl::from_str("file://rel_repo.git").is_err()); + } + + #[test] + fn bad_scheme_02() { + assert!(GitUrl::from_str("file:///abs_repo.git").is_err()); + } + + #[test] + fn bad_scheme_03() { + assert!(GitUrl::from_str("ssh://example.com/repo.git").is_err()); + } + + #[test] + fn bad_scheme_04() { + assert!(GitUrl::from_str("http://example.com/repo.git").is_err()); + } + + #[test] + fn bad_query_params() { + assert!(GitUrl::from_str("https://example.com/repo.git?admin=1").is_err()); + } + + #[test] + fn ok_empty_path_01() { + assert_eq!( + GitUrl::from_str("https://example.com").unwrap().to_path_buf(), + Path::new("https/example.com") + ) + } + + #[test] + fn ok_empty_path_02() { + assert_eq!( + GitUrl::from_str("https://example.com/").unwrap().to_path_buf(), + Path::new("https/example.com") + ) + } + + #[test] + fn ok_01() { + assert_eq!( + GitUrl::from_str("https://github.com/praetorian-inc/noseyparker.git").unwrap().to_path_buf(), + Path::new("https/github.com/praetorian-inc/noseyparker.git") + ); + } + + #[test] + fn ok_relpath_01() { + assert_eq!( + GitUrl::from_str("https://example.com/../boom.git").unwrap().to_path_buf(), + Path::new("https/example.com/boom.git") + ); + } + + #[test] + fn ok_relpath_02() { + assert_eq!( + GitUrl::from_str("https://example.com/root/../boom.git").unwrap().to_path_buf(), + Path::new("https/example.com/boom.git") + ); + } + + #[test] + fn ok_relpath_03() { + assert_eq!( + GitUrl::from_str("https://example.com/root/..").unwrap().to_path_buf(), + Path::new("https/example.com/") + ); + } +} diff --git a/src/github.rs b/src/github.rs index 2d129304f..f59bb034c 100644 --- a/src/github.rs +++ b/src/github.rs @@ -44,7 +44,7 @@ pub fn enumerate_repo_urls(repo_specifiers: &RepoSpecifiers) -> anyhow::Result, Error>(repo_urls) + Ok(repo_urls) // ::, Error>(repo_urls) }); match result { diff --git a/src/input_enumerator.rs b/src/input_enumerator.rs index a110d2f05..3275ca9aa 100644 --- a/src/input_enumerator.rs +++ b/src/input_enumerator.rs @@ -187,10 +187,6 @@ impl FilesystemEnumerator { pub const DEFAULT_FOLLOW_LINKS: bool = false; pub fn new>(inputs: &[T]) -> Result { - if inputs.is_empty() { - bail!("No root inputs provided"); - } - let mut builder = WalkBuilder::new(&inputs[0]); for input in &inputs[1..] { builder.add(input); diff --git a/src/lib.rs b/src/lib.rs index c1dd69d68..f55471089 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ pub mod bstring_escape; pub mod datastore; pub mod defaults; pub mod git_binary; +pub mod git_url; pub mod github; pub mod input_enumerator; pub mod location; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6861079c2..f315b57fa 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -9,7 +9,7 @@ pub use assert_cmd::prelude::*; pub use assert_fs::prelude::*; pub use assert_fs::{fixture::ChildPath, TempDir}; pub use insta::{assert_display_snapshot, assert_json_snapshot, assert_snapshot, with_settings}; -pub use predicates::str::RegexPredicate; +pub use predicates::str::{RegexPredicate, is_empty}; pub use pretty_assertions::{assert_eq, assert_ne}; pub use std::path::Path; pub use std::process::Command; @@ -38,7 +38,7 @@ macro_rules! assert_cmd_snapshot { macro_rules! noseyparker { ( $( $arg:expr ),* ) => { { - let mut cmd = common::noseyparker(); + let mut cmd = noseyparker_cmd(); $( cmd.arg($arg); )* @@ -59,6 +59,9 @@ macro_rules! noseyparker_failure { ( $( $arg:expr ),* ) => { noseyparker!($( $arg ),*).assert().failure() } } +// make macros easily visible to other modules +pub use {noseyparker, noseyparker_success, noseyparker_failure, assert_cmd_snapshot}; + lazy_static! { static ref NOSEYPARKER: escargot::CargoRun = escargot::CargoBuild::new() .bin("noseyparker") @@ -77,7 +80,7 @@ lazy_static! { } /// Build a `Command` for the `noseyparker` crate binary. -pub fn noseyparker() -> Command { +pub fn noseyparker_cmd() -> Command { // Command::cargo_bin("noseyparker").expect("noseyparker should be executable") NOSEYPARKER.command() } diff --git a/tests/scan.rs b/tests/scan.rs deleted file mode 100644 index bb4c44968..000000000 --- a/tests/scan.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! Tests for Nosey Parker `scan` command - -mod common; -use common::*; - -#[test] -fn scan_emptydir() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_dir("empty_dir"); - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) - .stdout(match_nothing_scanned()); -} - -#[test] -fn scan_datastore_argorder() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_dir("empty_dir"); - noseyparker_success!("scan", input.path(), "--datastore", scan_env.dspath()) - .stdout(match_nothing_scanned()); -} - -#[test] -fn scan_datastore_short() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_dir("empty_dir"); - noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) - .stdout(match_nothing_scanned()); -} - -#[test] -fn scan_datastore_envvar() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_dir("empty_dir"); - noseyparker!("scan", input.path()) - .env("NP_DATASTORE", scan_env.dspath()) - .assert() - .success() - .stdout(match_nothing_scanned()); -} - -#[test] -fn scan_emptyfile() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_file("empty_file"); - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) - .stdout(match_scan_stats("0B", 1, 0, 0)); -} - -#[test] -fn scan_emptyfiles() { - let scan_env = ScanEnv::new(); - let input1 = scan_env.input_file("empty_file1"); - let input2 = scan_env.input_file("empty_file2"); - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input1.path(), input2.path()) - .stdout(match_scan_stats("0B", 2, 0, 0)); -} - -#[test] -fn scan_file_symlink() { - let scan_env = ScanEnv::new(); - let empty_file = scan_env.input_file("empty_file"); - let input = scan_env.child("empty_file_link"); - input.symlink_to_file(empty_file).unwrap(); - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) - .stdout(match_nothing_scanned()); -} - -#[test] -fn scan_file_maxsize() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_file("bigfile.dat"); - input.write_binary(&[b'a'; 1024 * 1024 * 10]).unwrap(); - - // By default the input file gets scanned - noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) - .stdout(match_scan_stats("10.00 MiB", 1, 0, 0)); - - // With a restricted max file size, the file is not scanned - noseyparker_success!( - "scan", - "--datastore", - scan_env.dspath(), - input.path(), - "--max-file-size", - "5" - ) - .stdout(match_nothing_scanned()); - - // Also check for alternatively-spelled versions of a couple arguments - noseyparker_success!( - "scan", - format!("-d={}", scan_env.dspath().display()), - "--max-file-size=5.00", - input.path() - ) - .stdout(match_nothing_scanned()); -} - -#[cfg(unix)] -#[test] -fn scan_unreadable_file() { - use std::fs::{File, Permissions}; - use std::os::unix::fs::PermissionsExt; - - let scan_env = ScanEnv::new(); - let input = scan_env.input_file_with_secret("input.txt"); - // n.b. file value explicitly unnamed so it gets dropped - File::open(input.path()) - .unwrap() - .set_permissions(Permissions::from_mode(0o000)) - .unwrap(); - assert!(std::fs::read_to_string(input.path()).is_err()); - - noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) - .stdout(is_match("ERROR.*: Failed to load blob from .*: Permission denied")) - .stdout(match_nothing_scanned()); -} diff --git a/tests/scan_workflow.rs b/tests/scan_workflow.rs deleted file mode 100644 index 448782440..000000000 --- a/tests/scan_workflow.rs +++ /dev/null @@ -1,30 +0,0 @@ -//! Tests for a complete Nosey Parker scanning workflow - -mod common; -use common::*; - -#[test] -fn scan_secrets1() { - let scan_env = ScanEnv::new(); - let input = scan_env.input_file_with_secret("input.txt"); - - noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) - .stdout(match_scan_stats("81B", 1, 1, 1)); - - assert_cmd_snapshot!(noseyparker_success!("summarize", "-d", scan_env.dspath())); - - with_settings!({ - filters => vec![ - (r"(?m)^(\s*File: ).*$", r"$1 ") - ], - }, { - assert_cmd_snapshot!(noseyparker_success!("report", "-d", scan_env.dspath())); - }); - - - let cmd = noseyparker_success!("report", "-d", scan_env.dspath(), "--format=json"); - let json_output: serde_json::Value = serde_json::from_slice(&cmd.get_output().stdout).unwrap(); - assert_json_snapshot!(json_output, { - "[].matches[].provenance.path" => "/input.txt" - }); -} diff --git a/tests/snapshots/github__github_repos_list_noargs-2.snap b/tests/snapshots/github__github_repos_list_noargs-2.snap deleted file mode 100644 index 2caca4ffb..000000000 --- a/tests/snapshots/github__github_repos_list_noargs-2.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/github.rs -expression: stdout ---- - diff --git a/tests/snapshots/github__github_repos_list_org_badtoken-2.snap b/tests/snapshots/github__github_repos_list_org_badtoken-2.snap deleted file mode 100644 index 2caca4ffb..000000000 --- a/tests/snapshots/github__github_repos_list_org_badtoken-2.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/github.rs -expression: stdout ---- - diff --git a/tests/snapshots/github__github_repos_list_user_badtoken-2.snap b/tests/snapshots/github__github_repos_list_user_badtoken-2.snap deleted file mode 100644 index 2caca4ffb..000000000 --- a/tests/snapshots/github__github_repos_list_user_badtoken-2.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/github.rs -expression: stdout ---- - diff --git a/tests/snapshots/help__help-3.snap b/tests/snapshots/help__help-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_datastore-3.snap b/tests/snapshots/help__help_datastore-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_datastore-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_github-3.snap b/tests/snapshots/help__help_github-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_github-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_github_repos-3.snap b/tests/snapshots/help__help_github_repos-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_github_repos-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_github_repos_short-3.snap b/tests/snapshots/help__help_github_repos_short-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_github_repos_short-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_github_repos_short.snap b/tests/snapshots/help__help_github_repos_short.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_github_repos_short.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__help_github_short-3.snap b/tests/snapshots/help__help_github_short-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_github_short-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_github_short.snap b/tests/snapshots/help__help_github_short.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_github_short.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__help_report-3.snap b/tests/snapshots/help__help_report-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_report-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_report.snap b/tests/snapshots/help__help_report.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_report.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__help_report_short-3.snap b/tests/snapshots/help__help_report_short-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_report_short-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_report_short.snap b/tests/snapshots/help__help_report_short.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_report_short.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__help_rules-3.snap b/tests/snapshots/help__help_rules-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_rules-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_rules.snap b/tests/snapshots/help__help_rules.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_rules.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__help_scan-3.snap b/tests/snapshots/help__help_scan-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_scan-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_scan.snap b/tests/snapshots/help__help_scan.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_scan.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__help_scan_short-3.snap b/tests/snapshots/help__help_scan_short-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_scan_short-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_scan_short.snap b/tests/snapshots/help__help_scan_short.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_scan_short.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__help_short-3.snap b/tests/snapshots/help__help_short-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_short-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_short.snap b/tests/snapshots/help__help_short.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_short.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__help_summarize-3.snap b/tests/snapshots/help__help_summarize-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_summarize-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_summarize.snap b/tests/snapshots/help__help_summarize.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_summarize.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__help_summarize_short-3.snap b/tests/snapshots/help__help_summarize_short-3.snap deleted file mode 100644 index 30e5d1b37..000000000 --- a/tests/snapshots/help__help_summarize_short-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stderr ---- - diff --git a/tests/snapshots/help__help_summarize_short.snap b/tests/snapshots/help__help_summarize_short.snap deleted file mode 100644 index f892fabaa..000000000 --- a/tests/snapshots/help__help_summarize_short.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/help__no_args-2.snap b/tests/snapshots/help__no_args-2.snap deleted file mode 100644 index fb95b1a5b..000000000 --- a/tests/snapshots/help__no_args-2.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/help.rs -expression: stdout ---- - diff --git a/tests/snapshots/scan_workflow__scan_secrets1-3.snap b/tests/snapshots/scan_workflow__scan_secrets1-3.snap deleted file mode 100644 index 2ed81a9af..000000000 --- a/tests/snapshots/scan_workflow__scan_secrets1-3.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/scan_workflow.rs -expression: stderr ---- - diff --git a/tests/snapshots/scan_workflow__scan_secrets1-4.snap b/tests/snapshots/scan_workflow__scan_secrets1-4.snap deleted file mode 100644 index 20c942991..000000000 --- a/tests/snapshots/scan_workflow__scan_secrets1-4.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/scan_workflow.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/scan_workflow__scan_secrets1-6.snap b/tests/snapshots/scan_workflow__scan_secrets1-6.snap deleted file mode 100644 index 2ed81a9af..000000000 --- a/tests/snapshots/scan_workflow__scan_secrets1-6.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/scan_workflow.rs -expression: stderr ---- - diff --git a/tests/snapshots/scan_workflow__scan_secrets1.snap b/tests/snapshots/scan_workflow__scan_secrets1.snap deleted file mode 100644 index 20c942991..000000000 --- a/tests/snapshots/scan_workflow__scan_secrets1.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: tests/scan_workflow.rs -expression: status ---- -exit status: 0 diff --git a/tests/snapshots/test_noseyparker_github__github_repos_list_noargs-2.snap b/tests/snapshots/test_noseyparker_github__github_repos_list_noargs-2.snap new file mode 100644 index 000000000..bdf77b6a9 --- /dev/null +++ b/tests/snapshots/test_noseyparker_github__github_repos_list_noargs-2.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_github.rs +expression: stdout +--- + diff --git a/tests/snapshots/github__github_repos_list_noargs-3.snap b/tests/snapshots/test_noseyparker_github__github_repos_list_noargs-3.snap similarity index 59% rename from tests/snapshots/github__github_repos_list_noargs-3.snap rename to tests/snapshots/test_noseyparker_github__github_repos_list_noargs-3.snap index b407bc1c9..6b49c5359 100644 --- a/tests/snapshots/github__github_repos_list_noargs-3.snap +++ b/tests/snapshots/test_noseyparker_github__github_repos_list_noargs-3.snap @@ -1,5 +1,5 @@ --- -source: tests/github.rs +source: tests/test_noseyparker_github.rs expression: stderr --- Error: No repositories specified diff --git a/tests/snapshots/test_noseyparker_github__github_repos_list_noargs.snap b/tests/snapshots/test_noseyparker_github__github_repos_list_noargs.snap new file mode 100644 index 000000000..c6adef26a --- /dev/null +++ b/tests/snapshots/test_noseyparker_github__github_repos_list_noargs.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_github.rs +expression: status +--- +exit status: 2 diff --git a/tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken-2.snap b/tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken-2.snap new file mode 100644 index 000000000..bdf77b6a9 --- /dev/null +++ b/tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken-2.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_github.rs +expression: stdout +--- + diff --git a/tests/snapshots/github__github_repos_list_org_badtoken-3.snap b/tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken-3.snap similarity index 85% rename from tests/snapshots/github__github_repos_list_org_badtoken-3.snap rename to tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken-3.snap index de8cdb044..f7b88893a 100644 --- a/tests/snapshots/github__github_repos_list_org_badtoken-3.snap +++ b/tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken-3.snap @@ -1,5 +1,5 @@ --- -source: tests/github.rs +source: tests/test_noseyparker_github.rs expression: stderr --- Error: error making request: HTTP status client error (401 Unauthorized) for url (https://api.github.com/rate_limit) diff --git a/tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken.snap b/tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken.snap new file mode 100644 index 000000000..c6adef26a --- /dev/null +++ b/tests/snapshots/test_noseyparker_github__github_repos_list_org_badtoken.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_github.rs +expression: status +--- +exit status: 2 diff --git a/tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken-2.snap b/tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken-2.snap new file mode 100644 index 000000000..bdf77b6a9 --- /dev/null +++ b/tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken-2.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_github.rs +expression: stdout +--- + diff --git a/tests/snapshots/github__github_repos_list_user_badtoken-3.snap b/tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken-3.snap similarity index 85% rename from tests/snapshots/github__github_repos_list_user_badtoken-3.snap rename to tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken-3.snap index de8cdb044..f7b88893a 100644 --- a/tests/snapshots/github__github_repos_list_user_badtoken-3.snap +++ b/tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken-3.snap @@ -1,5 +1,5 @@ --- -source: tests/github.rs +source: tests/test_noseyparker_github.rs expression: stderr --- Error: error making request: HTTP status client error (401 Unauthorized) for url (https://api.github.com/rate_limit) diff --git a/tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken.snap b/tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken.snap new file mode 100644 index 000000000..c6adef26a --- /dev/null +++ b/tests/snapshots/test_noseyparker_github__github_repos_list_user_badtoken.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_github.rs +expression: status +--- +exit status: 2 diff --git a/tests/snapshots/help__help-2.snap b/tests/snapshots/test_noseyparker_help__help-2.snap similarity index 97% rename from tests/snapshots/help__help-2.snap rename to tests/snapshots/test_noseyparker_help__help-2.snap index 9b2777150..fffbbb275 100644 --- a/tests/snapshots/help__help-2.snap +++ b/tests/snapshots/test_noseyparker_help__help-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Find secrets and sensitive information in textual data diff --git a/tests/snapshots/test_noseyparker_help__help-3.snap b/tests/snapshots/test_noseyparker_help__help-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_datastore.snap b/tests/snapshots/test_noseyparker_help__help.snap similarity index 51% rename from tests/snapshots/help__help_datastore.snap rename to tests/snapshots/test_noseyparker_help__help.snap index f892fabaa..fff8e6509 100644 --- a/tests/snapshots/help__help_datastore.snap +++ b/tests/snapshots/test_noseyparker_help__help.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: status --- exit status: 0 diff --git a/tests/snapshots/help__help_datastore-2.snap b/tests/snapshots/test_noseyparker_help__help_datastore-2.snap similarity index 96% rename from tests/snapshots/help__help_datastore-2.snap rename to tests/snapshots/test_noseyparker_help__help_datastore-2.snap index 438ef814b..4bbd9f4aa 100644 --- a/tests/snapshots/help__help_datastore-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_datastore-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Manage datastores diff --git a/tests/snapshots/test_noseyparker_help__help_datastore-3.snap b/tests/snapshots/test_noseyparker_help__help_datastore-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_datastore-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_github.snap b/tests/snapshots/test_noseyparker_help__help_datastore.snap similarity index 51% rename from tests/snapshots/help__help_github.snap rename to tests/snapshots/test_noseyparker_help__help_datastore.snap index f892fabaa..fff8e6509 100644 --- a/tests/snapshots/help__help_github.snap +++ b/tests/snapshots/test_noseyparker_help__help_datastore.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: status --- exit status: 0 diff --git a/tests/snapshots/help__help_github-2.snap b/tests/snapshots/test_noseyparker_help__help_github-2.snap similarity index 97% rename from tests/snapshots/help__help_github-2.snap rename to tests/snapshots/test_noseyparker_help__help_github-2.snap index 2586ed74f..6ed045a11 100644 --- a/tests/snapshots/help__help_github-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_github-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Interact with GitHub diff --git a/tests/snapshots/test_noseyparker_help__help_github-3.snap b/tests/snapshots/test_noseyparker_help__help_github-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_github-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help_github_repos.snap b/tests/snapshots/test_noseyparker_help__help_github.snap similarity index 51% rename from tests/snapshots/help__help_github_repos.snap rename to tests/snapshots/test_noseyparker_help__help_github.snap index f892fabaa..fff8e6509 100644 --- a/tests/snapshots/help__help_github_repos.snap +++ b/tests/snapshots/test_noseyparker_help__help_github.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: status --- exit status: 0 diff --git a/tests/snapshots/help__help_github_repos-2.snap b/tests/snapshots/test_noseyparker_help__help_github_repos-2.snap similarity index 96% rename from tests/snapshots/help__help_github_repos-2.snap rename to tests/snapshots/test_noseyparker_help__help_github_repos-2.snap index cca983e67..e9f18fbcc 100644 --- a/tests/snapshots/help__help_github_repos-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_github_repos-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Interact with GitHub repositories diff --git a/tests/snapshots/test_noseyparker_help__help_github_repos-3.snap b/tests/snapshots/test_noseyparker_help__help_github_repos-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_github_repos-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/help__help.snap b/tests/snapshots/test_noseyparker_help__help_github_repos.snap similarity index 51% rename from tests/snapshots/help__help.snap rename to tests/snapshots/test_noseyparker_help__help_github_repos.snap index f892fabaa..fff8e6509 100644 --- a/tests/snapshots/help__help.snap +++ b/tests/snapshots/test_noseyparker_help__help_github_repos.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: status --- exit status: 0 diff --git a/tests/snapshots/help__help_github_repos_short-2.snap b/tests/snapshots/test_noseyparker_help__help_github_repos_short-2.snap similarity index 94% rename from tests/snapshots/help__help_github_repos_short-2.snap rename to tests/snapshots/test_noseyparker_help__help_github_repos_short-2.snap index b56a910e3..fda06e14d 100644 --- a/tests/snapshots/help__help_github_repos_short-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_github_repos_short-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Interact with GitHub repositories diff --git a/tests/snapshots/test_noseyparker_help__help_github_repos_short-3.snap b/tests/snapshots/test_noseyparker_help__help_github_repos_short-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_github_repos_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_github_repos_short.snap b/tests/snapshots/test_noseyparker_help__help_github_repos_short.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_github_repos_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_github_short-2.snap b/tests/snapshots/test_noseyparker_help__help_github_short-2.snap similarity index 94% rename from tests/snapshots/help__help_github_short-2.snap rename to tests/snapshots/test_noseyparker_help__help_github_short-2.snap index c383d0793..e3e72c827 100644 --- a/tests/snapshots/help__help_github_short-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_github_short-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Interact with GitHub diff --git a/tests/snapshots/test_noseyparker_help__help_github_short-3.snap b/tests/snapshots/test_noseyparker_help__help_github_short-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_github_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_github_short.snap b/tests/snapshots/test_noseyparker_help__help_github_short.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_github_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_report-2.snap b/tests/snapshots/test_noseyparker_help__help_report-2.snap similarity index 97% rename from tests/snapshots/help__help_report-2.snap rename to tests/snapshots/test_noseyparker_help__help_report-2.snap index 0487a13ac..d51b3ed68 100644 --- a/tests/snapshots/help__help_report-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_report-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Report detailed scan findings diff --git a/tests/snapshots/test_noseyparker_help__help_report-3.snap b/tests/snapshots/test_noseyparker_help__help_report-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_report-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_report.snap b/tests/snapshots/test_noseyparker_help__help_report.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_report.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_report_short-2.snap b/tests/snapshots/test_noseyparker_help__help_report_short-2.snap similarity index 95% rename from tests/snapshots/help__help_report_short-2.snap rename to tests/snapshots/test_noseyparker_help__help_report_short-2.snap index 13d4d227a..51bd1cebd 100644 --- a/tests/snapshots/help__help_report_short-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_report_short-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Report detailed scan findings diff --git a/tests/snapshots/test_noseyparker_help__help_report_short-3.snap b/tests/snapshots/test_noseyparker_help__help_report_short-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_report_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_report_short.snap b/tests/snapshots/test_noseyparker_help__help_report_short.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_report_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_rules-2.snap b/tests/snapshots/test_noseyparker_help__help_rules-2.snap similarity index 96% rename from tests/snapshots/help__help_rules-2.snap rename to tests/snapshots/test_noseyparker_help__help_rules-2.snap index 7ccb741ed..6dae910a6 100644 --- a/tests/snapshots/help__help_rules-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_rules-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Manage rules diff --git a/tests/snapshots/test_noseyparker_help__help_rules-3.snap b/tests/snapshots/test_noseyparker_help__help_rules-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_rules-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_rules.snap b/tests/snapshots/test_noseyparker_help__help_rules.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_rules.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_scan-2.snap b/tests/snapshots/test_noseyparker_help__help_scan-2.snap similarity index 95% rename from tests/snapshots/help__help_scan-2.snap rename to tests/snapshots/test_noseyparker_help__help_scan-2.snap index 1f93258bd..fd933be0b 100644 --- a/tests/snapshots/help__help_scan-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_scan-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Scan content for secrets @@ -65,8 +65,11 @@ Input Specifier Options: [INPUT]... Path to a file, directory, or local Git repository to scan - --git-repo + --git-url URL of a Git repository to clone and scan + + Only https URLs without credentials, query parameters, or fragment identifiers are + supported. --github-organization Name of a GitHub organization to enumerate and scan diff --git a/tests/snapshots/test_noseyparker_help__help_scan-3.snap b/tests/snapshots/test_noseyparker_help__help_scan-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_scan-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_scan.snap b/tests/snapshots/test_noseyparker_help__help_scan.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_scan.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_scan_short-2.snap b/tests/snapshots/test_noseyparker_help__help_scan_short-2.snap similarity index 92% rename from tests/snapshots/help__help_scan_short-2.snap rename to tests/snapshots/test_noseyparker_help__help_scan_short-2.snap index 22bb0ec53..c73ee7ec8 100644 --- a/tests/snapshots/help__help_scan_short-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_scan_short-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Scan content for secrets @@ -14,7 +14,7 @@ Options: Input Specifier Options: [INPUT]... Path to a file, directory, or local Git repository to scan - --git-repo URL of a Git repository to clone and scan + --git-url URL of a Git repository to clone and scan --github-organization Name of a GitHub organization to enumerate and scan [aliases: github-org] --github-user Name of a GitHub user to enumerate and scan diff --git a/tests/snapshots/test_noseyparker_help__help_scan_short-3.snap b/tests/snapshots/test_noseyparker_help__help_scan_short-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_scan_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_scan_short.snap b/tests/snapshots/test_noseyparker_help__help_scan_short.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_scan_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_short-2.snap b/tests/snapshots/test_noseyparker_help__help_short-2.snap similarity index 95% rename from tests/snapshots/help__help_short-2.snap rename to tests/snapshots/test_noseyparker_help__help_short-2.snap index d122462ac..a5b67969f 100644 --- a/tests/snapshots/help__help_short-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_short-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Find secrets and sensitive information in textual data diff --git a/tests/snapshots/test_noseyparker_help__help_short-3.snap b/tests/snapshots/test_noseyparker_help__help_short-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_short.snap b/tests/snapshots/test_noseyparker_help__help_short.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_summarize-2.snap b/tests/snapshots/test_noseyparker_help__help_summarize-2.snap similarity index 97% rename from tests/snapshots/help__help_summarize-2.snap rename to tests/snapshots/test_noseyparker_help__help_summarize-2.snap index 05c65021b..4f78d35a9 100644 --- a/tests/snapshots/help__help_summarize-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_summarize-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Summarize scan findings diff --git a/tests/snapshots/test_noseyparker_help__help_summarize-3.snap b/tests/snapshots/test_noseyparker_help__help_summarize-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_summarize-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_summarize.snap b/tests/snapshots/test_noseyparker_help__help_summarize.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_summarize.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/help__help_summarize_short-2.snap b/tests/snapshots/test_noseyparker_help__help_summarize_short-2.snap similarity index 95% rename from tests/snapshots/help__help_summarize_short-2.snap rename to tests/snapshots/test_noseyparker_help__help_summarize_short-2.snap index 1f4192736..94814e59a 100644 --- a/tests/snapshots/help__help_summarize_short-2.snap +++ b/tests/snapshots/test_noseyparker_help__help_summarize_short-2.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stdout --- Summarize scan findings diff --git a/tests/snapshots/test_noseyparker_help__help_summarize_short-3.snap b/tests/snapshots/test_noseyparker_help__help_summarize_short-3.snap new file mode 100644 index 000000000..5532a9b38 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_summarize_short-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_help__help_summarize_short.snap b/tests/snapshots/test_noseyparker_help__help_summarize_short.snap new file mode 100644 index 000000000..fff8e6509 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__help_summarize_short.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/test_noseyparker_help__no_args-2.snap b/tests/snapshots/test_noseyparker_help__no_args-2.snap new file mode 100644 index 000000000..52033e0e9 --- /dev/null +++ b/tests/snapshots/test_noseyparker_help__no_args-2.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_help.rs +expression: stdout +--- + diff --git a/tests/snapshots/help__no_args-3.snap b/tests/snapshots/test_noseyparker_help__no_args-3.snap similarity index 95% rename from tests/snapshots/help__no_args-3.snap rename to tests/snapshots/test_noseyparker_help__no_args-3.snap index 0f8f5edb1..e1e81c267 100644 --- a/tests/snapshots/help__no_args-3.snap +++ b/tests/snapshots/test_noseyparker_help__no_args-3.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_help.rs expression: stderr --- Find secrets and sensitive information in textual data diff --git a/tests/snapshots/github__github_repos_list_org_badtoken.snap b/tests/snapshots/test_noseyparker_help__no_args.snap similarity index 51% rename from tests/snapshots/github__github_repos_list_org_badtoken.snap rename to tests/snapshots/test_noseyparker_help__no_args.snap index 2ee2c85f0..168cceb83 100644 --- a/tests/snapshots/github__github_repos_list_org_badtoken.snap +++ b/tests/snapshots/test_noseyparker_help__no_args.snap @@ -1,5 +1,5 @@ --- -source: tests/github.rs +source: tests/test_noseyparker_help.rs expression: status --- exit status: 2 diff --git a/tests/snapshots/scan_workflow__scan_secrets1-2.snap b/tests/snapshots/test_noseyparker_scan__scan_secrets1-2.snap similarity index 87% rename from tests/snapshots/scan_workflow__scan_secrets1-2.snap rename to tests/snapshots/test_noseyparker_scan__scan_secrets1-2.snap index f65b8e2ff..a948bb9e5 100644 --- a/tests/snapshots/scan_workflow__scan_secrets1-2.snap +++ b/tests/snapshots/test_noseyparker_scan__scan_secrets1-2.snap @@ -1,5 +1,5 @@ --- -source: tests/scan_workflow.rs +source: tests/test_noseyparker_scan.rs expression: stdout --- diff --git a/tests/snapshots/test_noseyparker_scan__scan_secrets1-3.snap b/tests/snapshots/test_noseyparker_scan__scan_secrets1-3.snap new file mode 100644 index 000000000..366a93027 --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__scan_secrets1-3.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stderr +--- + diff --git a/tests/snapshots/test_noseyparker_scan__scan_secrets1-4.snap b/tests/snapshots/test_noseyparker_scan__scan_secrets1-4.snap new file mode 100644 index 000000000..211940a44 --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__scan_secrets1-4.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/scan_workflow__scan_secrets1-5.snap b/tests/snapshots/test_noseyparker_scan__scan_secrets1-5.snap similarity index 86% rename from tests/snapshots/scan_workflow__scan_secrets1-5.snap rename to tests/snapshots/test_noseyparker_scan__scan_secrets1-5.snap index ff414d432..dab5a8450 100644 --- a/tests/snapshots/scan_workflow__scan_secrets1-5.snap +++ b/tests/snapshots/test_noseyparker_scan__scan_secrets1-5.snap @@ -1,5 +1,5 @@ --- -source: tests/scan_workflow.rs +source: tests/test_noseyparker_scan.rs expression: stdout --- Finding 1/1: AWS API Key diff --git a/tests/snapshots/test_noseyparker_scan__scan_secrets1-6.snap b/tests/snapshots/test_noseyparker_scan__scan_secrets1-6.snap new file mode 100644 index 000000000..366a93027 --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__scan_secrets1-6.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stderr +--- + diff --git a/tests/snapshots/scan_workflow__scan_secrets1-7.snap b/tests/snapshots/test_noseyparker_scan__scan_secrets1-7.snap similarity index 96% rename from tests/snapshots/scan_workflow__scan_secrets1-7.snap rename to tests/snapshots/test_noseyparker_scan__scan_secrets1-7.snap index 6dc6adac3..f84f90d11 100644 --- a/tests/snapshots/scan_workflow__scan_secrets1-7.snap +++ b/tests/snapshots/test_noseyparker_scan__scan_secrets1-7.snap @@ -1,5 +1,5 @@ --- -source: tests/scan_workflow.rs +source: tests/test_noseyparker_scan.rs expression: json_output --- [ diff --git a/tests/snapshots/test_noseyparker_scan__scan_secrets1.snap b/tests/snapshots/test_noseyparker_scan__scan_secrets1.snap new file mode 100644 index 000000000..211940a44 --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__scan_secrets1.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: status +--- +exit status: 0 diff --git a/tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme-2.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme-2.snap new file mode 100644 index 000000000..e3c80c878 --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme-2.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stdout +--- + diff --git a/tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme-3.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme-3.snap new file mode 100644 index 000000000..d1ccb02aa --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme-3.snap @@ -0,0 +1,8 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stderr +--- +error: invalid value 'file://example.com/nothere.git' for '--git-url ': only https URLs without credentials, query parameters, or fragment identifiers are supported + +For more information, try '--help'. + diff --git a/tests/snapshots/github__github_repos_list_noargs.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme.snap similarity index 51% rename from tests/snapshots/github__github_repos_list_noargs.snap rename to tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme.snap index 2ee2c85f0..12982090e 100644 --- a/tests/snapshots/github__github_repos_list_noargs.snap +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__file_scheme.snap @@ -1,5 +1,5 @@ --- -source: tests/github.rs +source: tests/test_noseyparker_scan.rs expression: status --- exit status: 2 diff --git a/tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme-2.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme-2.snap new file mode 100644 index 000000000..e3c80c878 --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme-2.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stdout +--- + diff --git a/tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme-3.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme-3.snap new file mode 100644 index 000000000..becd14b8e --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme-3.snap @@ -0,0 +1,8 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stderr +--- +error: invalid value 'http://example.com/nothere.git' for '--git-url ': only https URLs without credentials, query parameters, or fragment identifiers are supported + +For more information, try '--help'. + diff --git a/tests/snapshots/github__github_repos_list_user_badtoken.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme.snap similarity index 51% rename from tests/snapshots/github__github_repos_list_user_badtoken.snap rename to tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme.snap index 2ee2c85f0..12982090e 100644 --- a/tests/snapshots/github__github_repos_list_user_badtoken.snap +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__http_scheme.snap @@ -1,5 +1,5 @@ --- -source: tests/github.rs +source: tests/test_noseyparker_scan.rs expression: status --- exit status: 2 diff --git a/tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme-2.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme-2.snap new file mode 100644 index 000000000..e3c80c878 --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme-2.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stdout +--- + diff --git a/tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme-3.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme-3.snap new file mode 100644 index 000000000..49701e6bf --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme-3.snap @@ -0,0 +1,8 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stderr +--- +error: invalid value 'nothere.git' for '--git-url ': only https URLs without credentials, query parameters, or fragment identifiers are supported + +For more information, try '--help'. + diff --git a/tests/snapshots/help__no_args.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme.snap similarity index 51% rename from tests/snapshots/help__no_args.snap rename to tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme.snap index 9988d332e..12982090e 100644 --- a/tests/snapshots/help__no_args.snap +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__no_scheme.snap @@ -1,5 +1,5 @@ --- -source: tests/help.rs +source: tests/test_noseyparker_scan.rs expression: status --- exit status: 2 diff --git a/tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme-2.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme-2.snap new file mode 100644 index 000000000..e3c80c878 --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme-2.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stdout +--- + diff --git a/tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme-3.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme-3.snap new file mode 100644 index 000000000..a8074baec --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme-3.snap @@ -0,0 +1,8 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: stderr +--- +error: invalid value 'ssh://example.com/nothere.git' for '--git-url ': only https URLs without credentials, query parameters, or fragment identifiers are supported + +For more information, try '--help'. + diff --git a/tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme.snap b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme.snap new file mode 100644 index 000000000..12982090e --- /dev/null +++ b/tests/snapshots/test_noseyparker_scan__test_scan_git_url__ssh_scheme.snap @@ -0,0 +1,5 @@ +--- +source: tests/test_noseyparker_scan.rs +expression: status +--- +exit status: 2 diff --git a/tests/github.rs b/tests/test_noseyparker_github.rs similarity index 97% rename from tests/github.rs rename to tests/test_noseyparker_github.rs index 6fca02fbf..790466dc6 100644 --- a/tests/github.rs +++ b/tests/test_noseyparker_github.rs @@ -1,6 +1,6 @@ //! Tests for Nosey Parker `github` command -pub use pretty_assertions::{assert_eq, assert_ne}; +use pretty_assertions::assert_eq; mod common; use common::*; @@ -12,7 +12,7 @@ fn github_repos_list_noargs() { #[test] fn github_repos_list_org_badtoken() { - let cmd = noseyparker() + let cmd = noseyparker!() .args(&["github", "repos", "list", "--org", "praetorian-inc"]) .env("NP_GITHUB_TOKEN", "hahabogus") .assert() @@ -22,7 +22,7 @@ fn github_repos_list_org_badtoken() { #[test] fn github_repos_list_user_badtoken() { - let cmd = noseyparker() + let cmd = noseyparker!() .args(&["github", "repos", "list", "--user", "octocat"]) .env("NP_GITHUB_TOKEN", "hahabogus") .assert() diff --git a/tests/help.rs b/tests/test_noseyparker_help.rs similarity index 100% rename from tests/help.rs rename to tests/test_noseyparker_help.rs diff --git a/tests/test_noseyparker_scan.rs b/tests/test_noseyparker_scan.rs new file mode 100644 index 000000000..9c90af1a4 --- /dev/null +++ b/tests/test_noseyparker_scan.rs @@ -0,0 +1,228 @@ +//! Tests for Nosey Parker `scan` command + +mod common; +use common::*; + +#[test] +fn scan_emptydir() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_dir("empty_dir"); + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_datastore_argorder() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_dir("empty_dir"); + noseyparker_success!("scan", input.path(), "--datastore", scan_env.dspath()) + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_datastore_short() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_dir("empty_dir"); + noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_datastore_envvar() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_dir("empty_dir"); + noseyparker!("scan", input.path()) + .env("NP_DATASTORE", scan_env.dspath()) + .assert() + .success() + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_emptyfile() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_file("empty_file"); + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) + .stdout(match_scan_stats("0B", 1, 0, 0)); +} + +#[test] +fn scan_emptyfiles() { + let scan_env = ScanEnv::new(); + let input1 = scan_env.input_file("empty_file1"); + let input2 = scan_env.input_file("empty_file2"); + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input1.path(), input2.path()) + .stdout(match_scan_stats("0B", 2, 0, 0)); +} + +#[test] +fn scan_file_symlink() { + let scan_env = ScanEnv::new(); + let empty_file = scan_env.input_file("empty_file"); + let input = scan_env.child("empty_file_link"); + input.symlink_to_file(empty_file).unwrap(); + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) + .stdout(match_nothing_scanned()); +} + +#[test] +fn scan_file_maxsize() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_file("bigfile.dat"); + input.write_binary(&[b'a'; 1024 * 1024 * 10]).unwrap(); + + // By default the input file gets scanned + noseyparker_success!("scan", "--datastore", scan_env.dspath(), input.path()) + .stdout(match_scan_stats("10.00 MiB", 1, 0, 0)); + + // With a restricted max file size, the file is not scanned + noseyparker_success!( + "scan", + "--datastore", + scan_env.dspath(), + input.path(), + "--max-file-size", + "5" + ) + .stdout(match_nothing_scanned()); + + // Also check for alternatively-spelled versions of a couple arguments + noseyparker_success!( + "scan", + format!("-d={}", scan_env.dspath().display()), + "--max-file-size=5.00", + input.path() + ) + .stdout(match_nothing_scanned()); +} + +#[cfg(unix)] +#[test] +fn scan_unreadable_file() { + use std::fs::{File, Permissions}; + use std::os::unix::fs::PermissionsExt; + + let scan_env = ScanEnv::new(); + let input = scan_env.input_file_with_secret("input.txt"); + // n.b. file value explicitly unnamed so it gets dropped + File::open(input.path()) + .unwrap() + .set_permissions(Permissions::from_mode(0o000)) + .unwrap(); + assert!(std::fs::read_to_string(input.path()).is_err()); + + noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) + .stdout(is_match("ERROR.*: Failed to load blob from .*: Permission denied")) + .stdout(match_nothing_scanned()); +} + +/// Create an empty Git repo on the filesystem at `destination`. +fn create_empty_git_repo(destination: &Path) { + Command::new("git") + .arg("init") + .arg(destination) + .assert() + .success() + .stdout(is_match("^Initialized empty Git repository in .*")) + .stderr(is_empty()); +} + +#[test] +fn scan_git_emptyrepo() { + let scan_env = ScanEnv::new(); + + let repo = scan_env.input_dir("input_repo"); + create_empty_git_repo(repo.path()); + + let path = format!("file://{}", repo.display()); + noseyparker_success!("scan", "-d", scan_env.dspath(), path) + .stdout(is_match(r"(?m)^Scanned .* from \d+ blobs in .*; 0/0 new matches$")); +} + +mod test_scan_git_url { + use super::*; + + #[test] + fn https_nonexistent() { + let scan_env = ScanEnv::new(); + + let path = "https://example.com/nothere.git"; + noseyparker_failure!("scan", "-d", scan_env.dspath(), "--git-url", path) + .stdout(is_match(r"(?m)^Cloning into bare repository .*$")) + .stdout(is_match(r"(?m)^fatal: repository .* not found$")) + .stderr(is_match(r"(?m)^Error: No inputs to scan$")); + } + + // Test what happens when there is no `git` binary but it is needed + #[test] + fn git_binary_missing() { + let scan_env = ScanEnv::new(); + + let path = "https://github.com/praetorian-inc/noseyparker"; + noseyparker!("scan", "-d", scan_env.dspath(), "--git-url", path) + .env_clear() + .env("PATH", "/dev/null") + .assert() + .failure() + .stdout(is_match(r"Failed to clone .*: git execution failed:")) + .stderr(is_match(r"(?m)^Error: No inputs to scan$")); + } + + #[test] + fn ssh_scheme() { + let scan_env = ScanEnv::new(); + let path = "ssh://example.com/nothere.git"; + assert_cmd_snapshot!(noseyparker_failure!("scan", "-d", scan_env.dspath(), "--git-url", path)); + } + + #[test] + fn http_scheme() { + let scan_env = ScanEnv::new(); + let path = "http://example.com/nothere.git"; + assert_cmd_snapshot!(noseyparker_failure!("scan", "-d", scan_env.dspath(), "--git-url", path)); + } + + #[test] + fn file_scheme() { + let scan_env = ScanEnv::new(); + let path = "file://example.com/nothere.git"; + assert_cmd_snapshot!(noseyparker_failure!("scan", "-d", scan_env.dspath(), "--git-url", path)); + } + + #[test] + fn no_scheme() { + let scan_env = ScanEnv::new(); + let path = "nothere.git"; + assert_cmd_snapshot!(noseyparker_failure!("scan", "-d", scan_env.dspath(), "--git-url", path)); + } +} + +// TODO: add test for scanning with `--github-user` +// TODO: add test for scanning with `--github-org` +// TODO: add test for caching behavior of rescanning `--git-url` + +#[test] +fn scan_secrets1() { + let scan_env = ScanEnv::new(); + let input = scan_env.input_file_with_secret("input.txt"); + + noseyparker_success!("scan", "-d", scan_env.dspath(), input.path()) + .stdout(match_scan_stats("81B", 1, 1, 1)); + + assert_cmd_snapshot!(noseyparker_success!("summarize", "-d", scan_env.dspath())); + + with_settings!({ + filters => vec![ + (r"(?m)^(\s*File: ).*$", r"$1 ") + ], + }, { + assert_cmd_snapshot!(noseyparker_success!("report", "-d", scan_env.dspath())); + }); + + + let cmd = noseyparker_success!("report", "-d", scan_env.dspath(), "--format=json"); + let json_output: serde_json::Value = serde_json::from_slice(&cmd.get_output().stdout).unwrap(); + assert_json_snapshot!(json_output, { + "[].matches[].provenance.path" => "/input.txt" + }); +} From 941610c85053a1b8a1d4ea5f3fa8edb6cc9b0a50 Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Thu, 16 Feb 2023 17:48:42 -0500 Subject: [PATCH 12/15] Fix GitHub Actions --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9f322902..3b97bcc3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,12 +47,13 @@ jobs: if: matrix.os == 'ubuntu-22.04' run: | sudo apt-get install -y --no-install-recommends \ - libhyperscan-dev + libhyperscan-dev \ + pkg-config - name: Install system dependencies (macOS) if: matrix.os == 'macos-12' run: | - brew install hyperscan + brew install hyperscan pkg-config - name: Install Rust toolchain id: install-rust-toolchain From a62a8aca495f1e20aa2820f4b48fcd5d95b9954c Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Thu, 16 Feb 2023 17:54:17 -0500 Subject: [PATCH 13/15] Fix git tests that fail on different platforms --- tests/test_noseyparker_scan.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_noseyparker_scan.rs b/tests/test_noseyparker_scan.rs index 9c90af1a4..edcf5bfdd 100644 --- a/tests/test_noseyparker_scan.rs +++ b/tests/test_noseyparker_scan.rs @@ -120,10 +120,11 @@ fn scan_unreadable_file() { fn create_empty_git_repo(destination: &Path) { Command::new("git") .arg("init") + .arg("-q") .arg(destination) .assert() .success() - .stdout(is_match("^Initialized empty Git repository in .*")) + .stdout(is_empty()) .stderr(is_empty()); } From abef716ed69e9bc3c35373ed3a21701fe131e73b Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Thu, 16 Feb 2023 19:41:32 -0500 Subject: [PATCH 14/15] Update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76831afdb..e4aa49034 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased ### Additions -- Nosey Parker now has rudimentary support for enumerating repositories from GitHub users and organizations. +- The `scan` command can now be given Git https URLs, GitHub usernames, and GitHub organization names as inputs, and will enumerate, clone, and scan as appropriate ([#14](https://github.com/praetorian-inc/noseyparker/issues/14)). + +- Nosey Parker now has rudimentary support for enumerating repositories from GitHub users and organizations ([#15](https://github.com/praetorian-inc/noseyparker/issues/15)). The new `github repos list` command uses the GitHub REST API to enumerate repositories belonging to one or more users or organizations. An optional GitHub Personal Access Token can be provided via the `NP_GITHUB_TOKEN` environment variable. From cab9bbed8016e680b8df8331db23093020e48e12 Mon Sep 17 00:00:00 2001 From: Brad Larsen Date: Thu, 16 Feb 2023 22:33:44 -0500 Subject: [PATCH 15/15] Fix CI to use debug tests for debug-mode builds --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b97bcc3a..1c9bdacea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: # spurious failures from rate limiting when testing Nosey Parker's github # enumeration capabilities. NP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: cargo test --locked --release --verbose + run: cargo test --locked --verbose - name: Check rules run: cargo run --locked -- rules check data/default/rules --warnings-as-errors