From 08f3fea7bad2f8479e17ef04ea7212dec52211c7 Mon Sep 17 00:00:00 2001 From: Arne Babenhauserheide Date: Sat, 12 Jun 2021 11:53:16 +0200 Subject: [PATCH 1/5] Add real node pitch black mitigation test --- src/freenet/node/LocationManager.java | 31 +- .../RealNodePitchBlackMitigationTest.java | 360 ++++++++++++++++++ 2 files changed, 383 insertions(+), 8 deletions(-) create mode 100644 src/freenet/node/simulator/RealNodePitchBlackMitigationTest.java diff --git a/src/freenet/node/LocationManager.java b/src/freenet/node/LocationManager.java index 1a42ea59d0e..03b58fc0eb7 100644 --- a/src/freenet/node/LocationManager.java +++ b/src/freenet/node/LocationManager.java @@ -17,6 +17,7 @@ import java.nio.file.Files; import java.security.MessageDigest; import java.text.DateFormat; +import java.time.Clock; import java.time.Duration; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -69,6 +70,8 @@ public class LocationManager implements ByteCounter { public static final String FOIL_PITCH_BLACK_ATTACK_PREFIX = "mitigate-pitch-black-attack-"; + public static long PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY = DAYS.toMillis(1); + public static long PITCH_BLACK_MITIGATION_STARTUP_DELAY = MINUTES.toMillis(10); public class MyCallback extends SendMessageOnErrorCallback { @@ -117,6 +120,7 @@ public void acknowledged() { final SwapRequestSender sender; final Node node; long timeLastSuccessfullySwapped; + public static Clock clockForTesting = Clock.systemDefaultZone(); public LocationManager(RandomSource r, Node node) { loc = r.nextDouble(); @@ -189,18 +193,18 @@ public void run() { @Override public void run() { - node.ticker.queueTimedJob(this, DAYS.toMillis(1)); + node.ticker.queueTimedJob(this, PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY); if (swappingDisabled()) { return; } - LocalDateTime now = LocalDateTime.now(); + LocalDateTime now = LocalDateTime.now(clockForTesting); String isoDateStringToday = DateTimeFormatter.ISO_DATE .format(now); String isoDateStringYesterday = DateTimeFormatter.ISO_DATE .format(now.minus(Duration.ofDays(1))); File[] previousInsertFromToday = node.userDir().dir() - .listFiles((file, name) -> name.startsWith(FOIL_PITCH_BLACK_ATTACK_PREFIX - + isoDateStringToday)); + .listFiles((file, name) -> name.startsWith(getPitchBlackPrefix( + isoDateStringToday))); HighLevelSimpleClient highLevelSimpleClient = node.clientCore.makeClient( RequestStarter.INTERACTIVE_PRIORITY_CLASS, true, @@ -211,13 +215,12 @@ public void run() { byte[] randomContentForKSK = new byte[20]; node.secureRandom.nextBytes(randomContentForKSK); String randomPart = Base64.encode(randomContentForKSK); - String nameForInsert = - FOIL_PITCH_BLACK_ATTACK_PREFIX + isoDateStringToday + "-" + randomPart; + String nameForInsert = getPitchBlackPrefix(isoDateStringToday + "-" + randomPart); tryToInsertPitchBlackCheck(highLevelSimpleClient, nameForInsert); } File[] foilPitchBlackStatusFiles = node.userDir().dir() - .listFiles((file, name) -> name.startsWith(FOIL_PITCH_BLACK_ATTACK_PREFIX)); + .listFiles((file, name) -> name.startsWith(getPitchBlackPrefix(""))); if (foilPitchBlackStatusFiles != null) { File[] successfulInsertFromYesterday = Arrays.stream(foilPitchBlackStatusFiles) .filter(file -> file.getName().contains(isoDateStringYesterday)) @@ -243,7 +246,11 @@ public void run() { } } } - }, MINUTES.toMillis(10)); + }, PITCH_BLACK_MITIGATION_STARTUP_DELAY); + } + + public String getPitchBlackPrefix(String middleSubstring) { + return FOIL_PITCH_BLACK_ATTACK_PREFIX + middleSubstring; } private void tryToRequestPitchBlackCheckFromYesterday( @@ -1592,6 +1599,14 @@ public Object[] getKnownLocations(long timestamp) { } } + public static void setClockForTesting(Clock clock) { + clockForTesting = clock; + } + + public static Clock getClockForTesting() { + return clockForTesting; + } + public static double[] extractLocs(PeerNode[] peers, boolean indicateBackoff) { double[] locs = new double[peers.length]; for(int i=0;i d))); + } + int newSwaps = LocationManager.swaps; + int totalStarted = LocationManager.startedSwaps; + int noSwaps = LocationManager.noSwaps; + System.err.println("Swaps: " + (newSwaps - lastSwaps)); + System.err.println("\nTotal swaps: Started*2: " + + totalStarted * 2 + + ", succeeded: " + + newSwaps + + ", last minute failures: " + + noSwaps + + + ", ratio " + + (double) noSwaps / (double) newSwaps + + ", early failures: " + + ((totalStarted * 2) - (noSwaps + newSwaps))); + System.err.println("This cycle ratio: " + ((double) (noSwaps - lastNoSwaps)) / ((double) ( + newSwaps + - lastSwaps))); + lastNoSwaps = noSwaps; + System.err.println("Swaps rejected (already locked): " + + LocationManager.swapsRejectedAlreadyLocked); + System.err.println("Swaps rejected (nowhere to go): " + + LocationManager.swapsRejectedNowhereToGo); + System.err.println("Swaps rejected (rate limit): " + LocationManager.swapsRejectedRateLimit); + System.err.println("Swaps rejected (recognized ID):" + + LocationManager.swapsRejectedRecognizedID); + System.err.println("Swaps failed:" + LocationManager.noSwaps); + System.err.println("Swaps succeeded:" + LocationManager.swaps); + + double totalSwapInterval = 0.0; + double totalSwapTime = 0.0; + for (int i = 0; i < nodes.length; i++) { + totalSwapInterval += nodes[i].lm.getSendSwapInterval(); + totalSwapTime += nodes[i].lm.getAverageSwapTime(); + } + System.err.println("Average swap time: " + (totalSwapTime / nodes.length)); + System.err.println("Average swap sender interval: " + (totalSwapInterval / nodes.length)); + + waitForAllConnected(nodes); + + lastSwaps = newSwaps; + // Do some (routed) test-pings + for (int i = 0; i < PINGS_PER_ITERATION; i++) { + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e1) { + } + try { + Node randomNode = nodes[random.nextInt(nodes.length)]; + Node randomNode2 = randomNode; + while (randomNode2 == randomNode) { + randomNode2 = nodes[random.nextInt(nodes.length)]; + } + double loc2 = randomNode2.getLocation(); + Logger.normal( + RealNodePitchBlackMitigationTest.class, + "Pinging " + randomNode2.getDarknetPortNumber() + " @ " + loc2 + " from " + randomNode + .getDarknetPortNumber() + " @ " + randomNode.getLocation()); + + int hopsTaken = randomNode.routedPing(loc2, randomNode2.getDarknetPubKeyHash()); + pings++; + if (hopsTaken < 0) { + failures++; + avg.report(0.0); + avg2.report(0.0); + double ratio = (double) successes / ((double) (failures + successes)); + System.err.println("Routed ping " + + pings + + " FAILED from " + + randomNode.getDarknetPortNumber() + + " to " + + randomNode2.getDarknetPortNumber() + + " (long:" + + ratio + + ", short:" + + avg.currentValue() + + ", vague:" + + avg2.currentValue() + + ')'); + } else { + totalHopsTaken += hopsTaken; + successes++; + avg.report(1.0); + avg2.report(1.0); + double ratio = (double) successes / ((double) (failures + successes)); + System.err.println("Routed ping " + + pings + + " success: " + + hopsTaken + + ' ' + + randomNode.getDarknetPortNumber() + + " to " + + randomNode2.getDarknetPortNumber() + + " (long:" + + ratio + + ", short:" + + avg.currentValue() + + ", vague:" + + avg2.currentValue() + + ')'); + } + } catch (Throwable t) { + Logger.error(RealNodePitchBlackMitigationTest.class, "Caught " + t, t); + } + } + System.err.println("Average path length for successful requests: " + + ((double) totalHopsTaken) / successes); + if (pings > MAX_PINGS || pings > MIN_PINGS && avg.currentValue() > accuracy && ((double) successes / ((double) (failures + + successes)) > accuracy)) { + System.err.println(); + System.err.println("Reached " + (accuracy * 100) + "% accuracy."); + System.err.println(); + System.err.println("Network size: " + nodes.length); + System.err.println("Maximum HTL: " + MAX_HTL); + System.err.println("Average path length for successful requests: " + + totalHopsTaken / successes); + System.err.println("Total started swaps: " + LocationManager.startedSwaps); + System.err.println("Total rejected swaps (already locked): " + + LocationManager.swapsRejectedAlreadyLocked); + System.err.println("Total swaps rejected (nowhere to go): " + + LocationManager.swapsRejectedNowhereToGo); + System.err.println("Total swaps rejected (rate limit): " + + LocationManager.swapsRejectedRateLimit); + System.err.println("Total swaps rejected (recognized ID):" + + LocationManager.swapsRejectedRecognizedID); + System.err.println("Total swaps failed:" + LocationManager.noSwaps); + System.err.println("Total swaps succeeded:" + LocationManager.swaps); + return; + } + } + System.exit(EXIT_PING_TARGET_NOT_REACHED); + } +} From 52b8552eea938626967879ee9a408a3b9d5013bd Mon Sep 17 00:00:00 2001 From: Arne Babenhauserheide Date: Sat, 12 Jun 2021 11:53:49 +0200 Subject: [PATCH 2/5] pitch-black-mitigation: prevent undamped oscillations of the pitch-black-mitigation --- src/freenet/node/LocationManager.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/freenet/node/LocationManager.java b/src/freenet/node/LocationManager.java index 03b58fc0eb7..66d4a68d957 100644 --- a/src/freenet/node/LocationManager.java +++ b/src/freenet/node/LocationManager.java @@ -278,10 +278,14 @@ private void tryToRequestPitchBlackCheckFromYesterday( try { sskFetchResult = highLevelSimpleClient.fetch(insertFromYesterday.getURI()); if (!Arrays.equals(expectedContent, sskFetchResult.asByteArray())) { + // if we received false data, this is definitely an attack: move there to provide a good node in the location switchLocationToDefendAgainstPitchBlackAttack(insertFromYesterday); } } catch (FetchException e) { - if (isRequestExceptionBecauseUriIsNotAvailable(e)) { + if (isRequestExceptionBecauseUriIsNotAvailable(e) && node.fastWeakRandom.nextBoolean()) { + // switch to the attacked location with only 50% probability, + // because it could be caused by the defensive swap of another node + // which made its current content inaccessible. switchLocationToDefendAgainstPitchBlackAttack(insertFromYesterday); } return; @@ -310,7 +314,10 @@ private void tryToRequestPitchBlackCheckFromYesterday( try { highLevelSimpleClient.fetch(calculatedChkUri); } catch (FetchException e) { - if (isRequestExceptionBecauseUriIsNotAvailable(e)) { + if (isRequestExceptionBecauseUriIsNotAvailable(e) && node.fastWeakRandom.nextBoolean()) { + // switch to the attacked location with only 50% probability, + // because it could be caused by the defensive swap of another node + // which made its current content inaccessible. try { switchLocationToDefendAgainstPitchBlackAttack(new ClientCHK(calculatedChkUri)); } catch (MalformedURLException exception) { From a60dbdf5ad6acf1575a91eedbb642df1b593443a Mon Sep 17 00:00:00 2001 From: Arne Babenhauserheide Date: Tue, 7 Sep 2021 23:06:16 +0200 Subject: [PATCH 3/5] document pitch black node test --- .../RealNodePitchBlackMitigationTest.java | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/freenet/node/simulator/RealNodePitchBlackMitigationTest.java b/src/freenet/node/simulator/RealNodePitchBlackMitigationTest.java index 8e542dfaf12..78cea15a447 100644 --- a/src/freenet/node/simulator/RealNodePitchBlackMitigationTest.java +++ b/src/freenet/node/simulator/RealNodePitchBlackMitigationTest.java @@ -28,16 +28,50 @@ import freenet.support.math.SimpleRunningAverage; /** - * @author amphibian + * @author ArneBab * - * Create a mesh of nodes and let them sort out their locations. + * This test spins uf Freenet nodes and simulates a pitch black attack and defense. + * It moves fake days forward to simulate the defense despite limited swapping speed. + * + * It spins up NUMBER_OF_NODES freenet nodes with DEEGREE. + * + * MIN_PINGS and MAX_PINGS give the minimum and maximum runtime. + * + * Adjust the variables NUMBER_OF_NODES, DEGREE, and PINGS_PER_ITERATION to adjust test parameters. + * + * Set PITCH_BLACK_ATTACK_MEAN_LOCATION and PITCH_BLACK_ATTACK_JITTER to select the location + * to attack. PITCH_BLACK_ATTACK_JITTER is necessary to prevent existind heuristics from deticting + * the naive attack with exactly one location. + * + * Set BETWEEN_PING_SLEEP_TIME to give the nodes time to swap between logging. + * + * PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY gives the time between triggering a mitigation + * (usually it is just one per day). It abvances the fake time by one day per period. + * + * Just grep the test output to get test results. Example Gnuplot calls to evaluate: + * + * set title "Average peer locations during pitch black mitigation" + * set xlabel "Time / Cycle" + * set ylabel "node / index" + * set cblabel "location / position in ring" + * plot "<(grep Cycle real-node-pitch-black-mitigation-test-results-11.log | grep ' node ' | sed 's/Cycle //;s/ node / /;s/: .*average=/ /;s/, .*$//;s/,/./g')" using 1:2:3 palette pt 5 ps 1.5 lw 1 title "RealNodePitchBlackMitigationTest" + * + * set title "Average path length of successful pings" + * set xlabel "Time / Cycle" + * set ylabel "average path lenth / hops" + * plot "<(grep 'Average path length' real-node-pitch-black-mitigation-test-results-11.log | sed 's/.*: //')" using 0:1 pt 5 ps 1.5 lw 1 title "RealNodePitchBlackMitigationTest" + * + * set title "Ping-Statistics" + * set xlabel "Time / Ping Number" + * set ylabel "fraction / unitless" + * set cblabel "path / hops needed" + * plot "<(grep 'Routed ping' real-node-pitch-black-mitigation-test-results-11.log | grep success | sed 's/Routed ping //;s/ success: / /g')" using 1:(($0+1)/$1):2 palette pt 3 ps 1 lw 1 title "succeeded", "<(grep 'Routed ping' real-node-pitch-black-mitigation-test-results-11.log | grep FAILED | sed 's/Routed ping //;s/FAILED from//')" using 1:(($0+1)/$1) pt 6 ps 1 lw 1 title "FAILED" * - * Then run some node-to-node searches. */ public class RealNodePitchBlackMitigationTest extends RealNodeTest { - static final int NUMBER_OF_NODES = 100; - static final int DEGREE = 10; + static final int NUMBER_OF_NODES = 300; + static final int DEGREE = 4; static final short MAX_HTL = (short) 10; static final boolean START_WITH_IDEAL_LOCATIONS = true; static final boolean FORCE_NEIGHBOUR_CONNECTIONS = true; From a4f8b05772aabee15d64f0fb48b62fdb53eea3b8 Mon Sep 17 00:00:00 2001 From: Arne Babenhauserheide Date: Tue, 7 Sep 2021 23:07:05 +0200 Subject: [PATCH 4/5] fix error-prone clock usage --- src/freenet/node/LocationManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/freenet/node/LocationManager.java b/src/freenet/node/LocationManager.java index 66d4a68d957..2a3f5f5be64 100644 --- a/src/freenet/node/LocationManager.java +++ b/src/freenet/node/LocationManager.java @@ -201,7 +201,7 @@ public void run() { String isoDateStringToday = DateTimeFormatter.ISO_DATE .format(now); String isoDateStringYesterday = DateTimeFormatter.ISO_DATE - .format(now.minus(Duration.ofDays(1))); + .format(now.minusDays(1)); File[] previousInsertFromToday = node.userDir().dir() .listFiles((file, name) -> name.startsWith(getPitchBlackPrefix( isoDateStringToday))); From 3eb6615b6e58b5669c5a51a66e517c07a5986110 Mon Sep 17 00:00:00 2001 From: Arne Babenhauserheide Date: Sun, 3 Oct 2021 22:22:23 +0200 Subject: [PATCH 5/5] Add note about RealNodePitchBlackMitigationTest.java to the simulator/readme.txt --- src/freenet/node/simulator/readme.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/freenet/node/simulator/readme.txt b/src/freenet/node/simulator/readme.txt index 8f11ab21325..614c373b63c 100644 --- a/src/freenet/node/simulator/readme.txt +++ b/src/freenet/node/simulator/readme.txt @@ -3,3 +3,5 @@ To run simulations, do something like: java -cp freenet.jar:freenet-ext.jar freenet.node.simulator.RealNodeProbeTest On Windows the classpath separator is ; instead of :. + +To test the pitch black mitigation, see RealNodePitchBlackMitigationTest.java