Skip to content

Commit

Permalink
Add and improve rules (#227)
Browse files Browse the repository at this point in the history
New rules:
- `Credentials in PostgreSQL Connection URI`
- `Django Secret Key`
- `PHPMailer Credentials`

Improved rules:
- `Credentials in ODBC  Connection String`

Other:
- Fixed a currently-benign bug in regex scanning that would result in matches being lost when patterns were compiled with the `SOM_LEFTMOST` option
- Adjusted logging span levels in scanning
  • Loading branch information
bradlarsen authored Oct 29, 2024
1 parent 30369fa commit 2598385
Show file tree
Hide file tree
Showing 14 changed files with 312 additions and 78 deletions.
9 changes: 7 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- The `/proc`, `/sys`, and `/dev` paths (special filesystems on Linux) are now ignored by default ([#225](https://github.com/praetorian-inc/noseyparker/pull/225)).
This suppresses many innocuous errors that would previously be seen when scanning the root filesystem of a Linux system.

- Lockfiles from a few languages (e.g., `Cargo.lock`, `Pipfile.lock`) are now ignored by default.
- Lockfiles from a few languages (e.g., `Cargo.lock`, `Pipfile.lock`, `go.sum`) are now ignored by default.

- The category metadata for `Age Recipient (X25519 public key)` and `ThingsBoard Access Token` has been expanded.
- Rules have been modified:
- `Age Recipient (X25519 public key)` and `ThingsBoard Access Token` have expanded category metdata.
- `Credentials in ODBC Connection String` detects more occurrences ([#227](https://github.com/praetorian-inc/noseyparker/pull/227)).

### Additions

- New rules have been added:

- `Credentials in PostgreSQL Connection URI` ([#227](https://github.com/praetorian-inc/noseyparker/pull/227))
- `Django Secret Key` ([#227](https://github.com/praetorian-inc/noseyparker/pull/227))
- `HTTP Basic Authentication`
- `HTTP Bearer Token`
- `PHPMailer Credentials` ([#227](https://github.com/praetorian-inc/noseyparker/pull/227))


## [v0.20.0](https://github.com/praetorian-inc/noseyparker/releases/v0.20.0) (2024-10-04)
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Nosey Parker is a command-line tool that finds secrets and sensitive information

**Key features:**
- It natively scans files, directories, and Git repository history
- It uses regular expression matching with a set of [147 patterns](crates/noseyparker/data/default/builtin/rules) chosen for high signal-to-noise based on experience and feedback from offensive security engagements
- It uses regular expression matching with a set of [150 patterns](crates/noseyparker/data/default/builtin/rules) chosen for high signal-to-noise based on experience and feedback from offensive security engagements
- It deduplicates its findings, grouping matches together that share the same secret, which in practice can reduce review burden by 100x or more compared to other tools
- It is fast: it can scan at hundreds of megabytes per second on a single core, and is able to scan 100GB of Linux kernel source history in less than 2 minutes on an older MacBook Pro
- It scales: it has scanned inputs as large as 20TiB during security engagements
Expand Down
26 changes: 16 additions & 10 deletions crates/noseyparker-cli/src/cmd_scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use tracing::{debug, debug_span, error, info, trace, warn};
use tracing::{debug, error, error_span, info, trace, warn};

use crate::{args, rule_loader::RuleLoader};

Expand Down Expand Up @@ -658,22 +658,28 @@ struct BlobProcessor<'a> {
impl<'a> BlobProcessor<'a> {
fn run(&mut self, provenance: ProvenanceSet, blob: Blob) -> Result<Option<DatastoreMessage>> {
let blob_id = blob.id.hex();
let _span = debug_span!("matcher", blob_id).entered();

let t1 = Instant::now();
let res = self.matcher.scan_blob(&blob, &provenance)?;
let scan_us = t1.elapsed().as_micros();
let _span = error_span!("matcher", blob_id, bytes = blob.len()).entered();

let (res, scan_us, scan_mbps) = if tracing::enabled!(tracing::Level::TRACE) {
let t1 = Instant::now();
let res = self.matcher.scan_blob(&blob, &provenance)?;
let t1e = t1.elapsed();
(res, t1e.as_micros(), blob.len() as f64 / 1024.0 / 1024.0 / t1e.as_secs_f64())
} else {
let res = self.matcher.scan_blob(&blob, &provenance)?;
(res, Default::default(), Default::default())
};

match res {
// blob already seen, but with no matches; nothing to do!
ScanResult::SeenSansMatches => {
trace!("({scan_us}us) blob already scanned with no matches");
trace!(us = scan_us, mbps = scan_mbps, status = "seen_nomatch");
Ok(None)
}

// blob already seen; all we need to do is record its provenance
ScanResult::SeenWithMatches => {
trace!("({scan_us}us) blob already scanned with matches");
trace!(us = scan_us, mbps = scan_mbps, status = "seen_match");
let metadata = BlobMetadata {
id: blob.id,
num_bytes: blob.len(),
Expand All @@ -685,7 +691,7 @@ impl<'a> BlobProcessor<'a> {

// blob has not been seen; need to record blob metadata, provenance, and matches
ScanResult::New(matches) => {
trace!("({scan_us}us) blob newly scanned; {} matches", matches.len());
trace!(us = scan_us, mbps = scan_mbps, status = "new", matches = matches.len());

let do_copy_blob = match self.copy_blobs {
args::CopyBlobsMode::All => true,
Expand Down Expand Up @@ -921,7 +927,7 @@ fn datastore_writer(
mut datastore: Datastore,
recv_ds: crossbeam_channel::Receiver<DatastoreMessage>,
) -> Result<(Datastore, u64, u64)> {
let _span = debug_span!("datastore", "{}", datastore.root_dir().display()).entered();
let _span = error_span!("datastore", "{}", datastore.root_dir().display()).entered();
let mut total_recording_time: std::time::Duration = Default::default();

let mut num_matches_added: u64 = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
source: crates/noseyparker-cli/tests/rules/mod.rs
expression: stdout
---
147 rules and 3 rulesets: no issues detected
150 rules and 3 rulesets: no issues detected
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,29 @@ expression: stdout
]
}
},
{
"id": "np.django.1",
"structural_id": "da64b83c14b6ed50fb3f644c8f4243e1c2e5d9f6",
"name": "Django Secret Key",
"syntax": {
"name": "Django Secret Key",
"id": "np.django.1",
"pattern": "(?x)\n\\#\\ SECURITY\\ WARNING:\\ keep\\ the\\ secret\\ key\\ used\\ in\\ production\\ secret! \\s*\n.{0,5} SECRET_KEY \\s* = \\s* r?[\"'] ([^\"'\\n]{5,100}) [\"']\n",
"examples": [
"# Quick-start development settings - unsuitable for production\n# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/\n\n# SECURITY WARNING: keep the secret key used in production secret!\nSECRET_KEY = 'django-insecure-_du9e^cmago!%(^+=gr@cu@v9-v7ulhbk2s3!w&39w4+n3*k*$'\n\n# SECURITY WARNING: don't run with debug turned on in production!\nDEBUG = True\n",
"# Quick-start development settings - unsuitable for production\n# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/\n\n# SECURITY WARNING: keep the secret key used in production secret!\n# SECRET_KEY = 'django-insecure-_du9e^cmago!%(^+=gr@cu@v9-v7ulhbk2s3!w&39w4+n3*k*$'\nSECRET_KEY = 'hmm'\n\n# SECURITY WARNING: don't run with debug turned on in production!\nDEBUG = True\n"
],
"negative_examples": [],
"references": [
"https://docs.djangoproject.com/en/5.1/ref/settings/#std-setting-SECRET_KEY",
"https://docs.djangoproject.com/en/5.1/topics/signing/"
],
"categories": [
"fuzzy",
"secret"
]
}
},
{
"id": "np.dockerhub.1",
"structural_id": "ee5a64a1fc638eb07dcae2cfd5a32bb3f583395b",
Expand Down Expand Up @@ -1695,6 +1718,7 @@ expression: stdout
],
"categories": [
"secret",
"fuzzy",
"generic"
]
}
Expand All @@ -1717,6 +1741,7 @@ expression: stdout
],
"categories": [
"secret",
"fuzzy",
"generic"
]
}
Expand Down Expand Up @@ -2292,12 +2317,12 @@ expression: stdout
},
{
"id": "np.odbc.1",
"structural_id": "630c1001441e63d74fea2a730a5196ee38ac818b",
"structural_id": "6acf132b0cc66853bd5da65ce86f680e7147c003",
"name": "Credentials in ODBC Connection String",
"syntax": {
"name": "Credentials in ODBC Connection String",
"id": "np.odbc.1",
"pattern": "(?x)(?i)\n(?: User | User\\ Id | UserId | Uid) \\s*=\\s* ([^\\s;]{3,100}) \\s* ;\n[\\ \\t]* .{0,10} [\\ \\t]* (?# possible extra stuff, e.g., string concatenation)\n(?: Password | Pwd) \\s*=\\s* ([^\\t\\ ;]{3,100}) \\s* (?: [;] | $)\n",
"pattern": "(?x)(?i)\n(?: User | User\\ Id | UserId | Uid) \\s*=\\s* ([^\\s;]{3,100}) \\s* ;\n[\\ \\t]* .{0,10} [\\ \\t]* (?# possible extra stuff, e.g., string concatenation)\n(?: Password | Pwd) \\s*=\\s* ([^\\t\\ ;]{3,100})\n\\s* (?: [;\"'] | $)\n",
"examples": [
"//Database Info\\r\\n\\t\\t\\t\\t\\t$host = \\\"localhost\\\";\\r\\n\\t\\t\\t\\t\\t$database = \\\"NHOHVA\\\";\\r\\n\\t\\t\\t\\t\\t$user = \\\"mg1021\\\"; $password = \\\"goodspec\\\";",
"//Database Info\\r\\n\\t\\t\\t\\t\\t$host = \\\"localhost\\\";\\r\\n\\t\\t\\t\\t\\t$database = \\\"NHOHVA\\\";\\r\\n\\t\\t\\t\\t\\t$user = \\\"mg1021\\\"; $password = goodspec;",
Expand All @@ -2307,7 +2332,8 @@ expression: stdout
"Data Source=190.190.200.100,1433;Network_library=DBMSSOCN;Initial Catalog=myDataBase;User ID=myUsername;Password=myPassword;",
"Provider=SQLNCLI;Server=myServerName,myPortNumber;Database=myDataBase;Uid=myUsername;Pwd=myPassword;",
" adoConn.Open(\"Provider=SQLOLEDB.1;User ID=specialbill_user; \" & \"Password =specialbill_user;Initial Catalog=SpecialBill_PROD;Data Source=uszdba01;\")",
"\"driver={SQL Server};server=(#{datastore['DBHOST']});database=#{datastore['DBNAME']};uid=#{datastore['DBUID']};pwd=#{datastore['DBPASSWORD']}\"\n"
"\"driver={SQL Server};server=(#{datastore['DBHOST']});database=#{datastore['DBNAME']};uid=#{datastore['DBUID']};pwd=#{datastore['DBPASSWORD']}\"\n",
"<add name=\"DevQATrucks\" connectionString=\"Server=hookupsqlqa1\\gpkqa; Database=YogurtDelay: User Id=kmwulfliq; password= sK!g2Ex_=jJ6Gx5v\" providerName=\"System.Data.SqlClient\" />"
],
"negative_examples": [
"def login(self, user = '', password = '', domain = ''):",
Expand Down Expand Up @@ -2479,6 +2505,54 @@ expression: stdout
]
}
},
{
"id": "np.phpmailer.1",
"structural_id": "d6cefbc0eb75afeabb5915dd45dcbf8697380120",
"name": "PHPMailer Credentials",
"syntax": {
"name": "PHPMailer Credentials",
"id": "np.phpmailer.1",
"pattern": "(?x)\n\\$mail->Host \\s* = \\s* '([^'\\n]{5,})'; \\s* (?: //.* )?\n(?: \\s* .* \\s* ){0,3}\n\\$mail->Username \\s* = \\s* '([^'\\n]{5,})'; \\s* (?: //.* )?\n(?: \\s* .* \\s* ){0,3}\n\\$mail->Password \\s* = \\s* '([^'\\n]{5,})';\n",
"examples": [
"//Server settings\n$mail->SMTPDebug = SMTP::DEBUG_SERVER; //Enable verbose debug output\n$mail->isSMTP(); //Send using SMTP\n$mail->Host = 'smtp.example.com'; //Set the SMTP server to send through\n$mail->SMTPAuth = true; //Enable SMTP authentication\n$mail->Username = '[email protected]'; //SMTP username\n$mail->Password = 'secret'; //SMTP password\n$mail->SMTPSecure = PHPMailer::ENCRYPTION_SMTPS; //Enable implicit TLS encryption\n$mail->Port = 465; //TCP port to connect to; use 587 if you have set `SMTPSecure = PHPMailer::ENCRYPTION_STARTTLS`\n",
"require 'PHPMailerAutoload.php';\n\nfunction SendMail($sub,$to,$msg)\n{\n $mail = new PHPMailer;\n $mail->isSMTP(); // Set mailer to use SMTP\n $mail->Host = 'smtp.gmail.com'; // Specify main and backup SMTP servers\n $mail->SMTPAuth = true; // Enable SMTP authentication\n $mail->SMTPSecure = 'tls'; // Enable encryption, 'ssl' also accepted\n $mail->Username = '[email protected]'; // SMTP username\n\n\n\n $mail->Password = 'un!techwhooah'; // SMTP password\n $mail->From = '[email protected]';\n $mail->FromName = 'Admin';\n"
],
"negative_examples": [],
"references": [
"https://github.com/PHPMailer/PHPMailer"
],
"categories": [
"fuzzy",
"secret"
]
}
},
{
"id": "np.postgres.1",
"structural_id": "d1b558227cf1fa521f2db96bac75c1914554d667",
"name": "Credentials in PostgreSQL Connection URI",
"syntax": {
"name": "Credentials in PostgreSQL Connection URI",
"id": "np.postgres.1",
"pattern": "(?x)\n(?: postgres | postgresql ) :// (?# URI scheme )\n ([a-zA-Z0-9.-~]{3,}) (?# username)\n: ([a-zA-Z0-9.-~]{3,}) (?# password)\n@ ([a-zA-Z0-9_.-]{3,} (?: :\\d{1,5})?) (?# hostname and port)\n(/[a-zA-Z0-9_.-]{2,}) (?# database)\n(?: \\? [a-zA-Z0-9.-~]+ = [a-zA-Z0-9.-~]+\n (?: & [a-zA-Z0-9.-~]+ = [a-zA-Z0-9.-~]+ )* )? (?# query params )\n(?: [^a-zA-Z0-9.-~] | $ )\n",
"examples": [
"\"REDSHIFT\": \"postgres://spot_app:Pseg2020@calling-mr-bones.c0qsadyxbf4k.us-east-1.redshift.amazonaws.com:5439/datalakespotprod\",",
"postgresql://user:secret@localhost/database",
"postgresql://user:secret@localhost/otherdb?connect_timeout=10&application_name=myapp"
],
"negative_examples": [
"postgresql://user:secret@[2001:db8::1234]/database",
"postgresql://user:secret@host1:123,user:secret@host2:456/somedb?target_session_attrs=any&application_name=myapp",
"postgresql:///mydb?host=localhost&port=5433&user=user&password=secret"
],
"references": [
"https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING"
],
"categories": [
"secret"
]
}
},
{
"id": "np.postman.1",
"structural_id": "92b0a519b4ad321547051c203e58ed828d2480e1",
Expand Down Expand Up @@ -3552,7 +3626,7 @@ expression: stdout
{
"id": "default",
"name": "Nosey Parker default rules",
"num_rules": 126
"num_rules": 129
},
{
"id": "np.assets",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ expression: stdout
np.digitalocean.1 DigitalOcean Application Access Token api, secret
np.digitalocean.2 DigitalOcean Personal Access Token api, secret
np.digitalocean.3 DigitalOcean Refresh Token api, secret
np.django.1 Django Secret Key fuzzy, secret
np.dockerhub.1 Docker Hub Personal Access Token api, secret
np.doppler.1 Doppler CLI Token api, secret
np.doppler.2 Doppler Personal Token api, secret
Expand Down Expand Up @@ -74,8 +75,8 @@ expression: stdout
np.grafana.2 Grafana Cloud API Token api, secret
np.grafana.3 Grafana Service Account Token api, secret
np.heroku.1 Heroku API Key api, fuzzy, secret
np.http.1 HTTP Basic Authentication generic, secret
np.http.2 HTTP Bearer Token generic, secret
np.http.1 HTTP Basic Authentication fuzzy, generic, secret
np.http.2 HTTP Bearer Token fuzzy, generic, secret
np.huggingface.1 HuggingFace User Access Token api, secret
np.jenkins.1 Jenkins Token or Crumb api, fuzzy, secret
np.jwt.1 JSON Web Token (base64url-encoded) api
Expand Down Expand Up @@ -107,6 +108,8 @@ expression: stdout
np.particleio.2 particle.io Access Token api, secret
np.pem.1 PEM-Encoded Private Key secret
np.pem.2 Base64-PEM-Encoded Private Key secret
np.phpmailer.1 PHPMailer Credentials fuzzy, secret
np.postgres.1 Credentials in PostgreSQL Connection URI secret
np.postman.1 Postman API Key api, secret
np.psexec.1 Credentials in PsExec fuzzy, secret
np.pwhash.1 Password Hash (md5crypt) hashed, secret
Expand Down Expand Up @@ -154,6 +157,6 @@ expression: stdout

Ruleset ID Ruleset Name Rules
─────────────────────────────────────────────────────────
default Nosey Parker default rules 126
default Nosey Parker default rules 129
np.assets Nosey Parker asset detection rules 15
np.hashes Nosey Parker password hash rules 6
39 changes: 39 additions & 0 deletions crates/noseyparker/data/default/builtin/rules/django.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
rules:

- name: Django Secret Key
id: np.django.1

# This identifies cryptographic signing secrets in configuration files generated by `django-admin startproject`.
pattern: |
(?x)
\#\ SECURITY\ WARNING:\ keep\ the\ secret\ key\ used\ in\ production\ secret! \s*
.{0,5} SECRET_KEY \s* = \s* r?["'] ([^"'\n]{5,100}) ["']
categories: [fuzzy, secret]

examples:
- |
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'django-insecure-_du9e^cmago!%(^+=gr@cu@v9-v7ulhbk2s3!w&39w4+n3*k*$'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
- |
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
# SECRET_KEY = 'django-insecure-_du9e^cmago!%(^+=gr@cu@v9-v7ulhbk2s3!w&39w4+n3*k*$'
SECRET_KEY = 'hmm'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
references:
- https://docs.djangoproject.com/en/5.1/ref/settings/#std-setting-SECRET_KEY
- https://docs.djangoproject.com/en/5.1/topics/signing/
4 changes: 2 additions & 2 deletions crates/noseyparker/data/default/builtin/rules/http.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ rules:
- https://datatracker.ietf.org/doc/html/rfc7617
- https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication

categories: [secret, generic]
categories: [secret, fuzzy, generic]


- name: HTTP Bearer Token
Expand All @@ -39,4 +39,4 @@ rules:
- https://datatracker.ietf.org/doc/html/rfc6750
- https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication

categories: [secret, generic]
categories: [secret, fuzzy, generic]
5 changes: 4 additions & 1 deletion crates/noseyparker/data/default/builtin/rules/odbc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ rules:
(?x)(?i)
(?: User | User\ Id | UserId | Uid) \s*=\s* ([^\s;]{3,100}) \s* ;
[\ \t]* .{0,10} [\ \t]* (?# possible extra stuff, e.g., string concatenation)
(?: Password | Pwd) \s*=\s* ([^\t\ ;]{3,100}) \s* (?: [;] | $)
(?: Password | Pwd) \s*=\s* ([^\t\ ;]{3,100})
\s* (?: [;"'] | $)
categories: [fuzzy, secret]

Expand All @@ -23,6 +24,8 @@ rules:
- |
"driver={SQL Server};server=(#{datastore['DBHOST']});database=#{datastore['DBNAME']};uid=#{datastore['DBUID']};pwd=#{datastore['DBPASSWORD']}"
- '<add name="DevQATrucks" connectionString="Server=hookupsqlqa1\gpkqa; Database=YogurtDelay: User Id=kmwulfliq; password= sK!g2Ex_=jJ6Gx5v" providerName="System.Data.SqlClient" />'

negative_examples:
- "def login(self, user = '', password = '', domain = ''):"
- |
Expand Down
Loading

0 comments on commit 2598385

Please sign in to comment.