diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..b6728c4d088 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,35 @@ +Security +======== + +Freenet requires different security considerations than other projects. + +Any vulnerability that can correlate your activity with easily observable behavior of your node is critical. + +This specifically means: + +- any way to crash Freenet when accessing some known content is serious. +- if you can get Freenet or the browser opening a site from Freenet to make a request to some clearnet server depending on the content being accessed, this is serious. + +More so if the vulnerability affects the **friend-to-friend** mode. + +There are known unfixable attacks against opennet (Sybil attacks cannot be prevented completely, only their impact reduced). + +There are no known unfixable identification-attacks against friend-to-friend mode, except if your friends' nodes attack you. + +Attacks we know about are detailed on the opennet attacks and the major attacks page: + +- [Attack in General](https://github.com/freenet/wiki/wiki/Major-Attacks) +- [Attacks against Opennet](https://github.com/freenet/wiki/wiki/Opennet-Attacks) + +Reporting Security Issues +------------------------- + +Please report security issues to security@freenetproject.org +encrypting to all PGP/gnupg keys from our [Keyring](https://freenetproject.org/assets/keyring.gpg). + +Please do not file public reports of security problems that could be +used to connect the pseudonyms of users with the nodes they run. If +you find those, please send them to the email address above so they +can be resolved and the fix released before the vulnerability gets +someone in danger. + diff --git a/src/freenet/client/ArchiveManager.java b/src/freenet/client/ArchiveManager.java index 579762da82d..911c50f3db7 100644 --- a/src/freenet/client/ArchiveManager.java +++ b/src/freenet/client/ArchiveManager.java @@ -361,7 +361,7 @@ public void run() { } } catch (IOException ioe) { throw new ArchiveFailureException("An IOE occured: "+ioe.getMessage(), ioe); - }finally { + } finally { Closer.close(is); } } diff --git a/src/freenet/clients/http/ContentFilterToadlet.java b/src/freenet/clients/http/ContentFilterToadlet.java index 8d4f5172c46..8c6afab994e 100644 --- a/src/freenet/clients/http/ContentFilterToadlet.java +++ b/src/freenet/clients/http/ContentFilterToadlet.java @@ -136,14 +136,20 @@ private HTMLNode createContent(PageMaker pageMaker, ToadletContext ctx) { // display in browser or save to disk filterForm.addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "result-handling", ResultHandling.DISPLAY.toString() }); - filterForm.addChild("#", l10n("displayResultLabel")); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "result-handling", ResultHandling.DISPLAY.toString(), "result-handling-display" }); + filterForm.addChild("label", + new String[] { "for" }, + new String[] { "result-handling-display" }, + l10n("displayResultLabel")); filterForm.addChild("br"); filterForm.addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "result-handling", ResultHandling.SAVE.toString() }); - filterForm.addChild("#", l10n("saveResultLabel")); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "result-handling", ResultHandling.SAVE.toString(), "result-handling-save" }); + filterForm.addChild("label", + new String[] { "for" }, + new String[] { "result-handling-save" }, + l10n("saveResultLabel")); filterForm.addChild("br"); filterForm.addChild("br"); diff --git a/src/freenet/clients/http/DiagnosticToadlet.java b/src/freenet/clients/http/DiagnosticToadlet.java index 269d5d069a4..a44a33d54bb 100644 --- a/src/freenet/clients/http/DiagnosticToadlet.java +++ b/src/freenet/clients/http/DiagnosticToadlet.java @@ -2,12 +2,16 @@ import java.io.File; import java.io.IOException; +import java.lang.management.*; import java.net.URI; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.Arrays; import java.util.Comparator; +import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.*; import freenet.client.HighLevelSimpleClient; import freenet.client.async.PersistenceDisabledException; @@ -29,6 +33,8 @@ import freenet.node.PeerNodeStatus; import freenet.node.RequestTracker; import freenet.node.Version; +import freenet.node.diagnostics.*; +import freenet.node.diagnostics.threads.*; import freenet.node.stats.DataStoreInstanceType; import freenet.node.stats.DataStoreStats; import freenet.node.stats.StatsNotAvailableException; @@ -410,19 +416,69 @@ public int compare(PeerNodeStatus firstNode, PeerNodeStatus secondNode) { textBuilder.append("\n"); // drawThreadPriorityStatsBox - textBuilder.append("Threads:\n"); - int[] activeThreadsByPriority = stats.getActiveThreadsByPriority(); - int[] waitingThreadsByPriority = stats.getWaitingThreadsByPriority(); - for(int i=0; i threads = threadSnapshot.getThreads(); + threads.sort(Comparator.comparing(NodeThreadInfo::getCpuTime).reversed()); + + sb.append(String.format("Threads (%d):%n", threads.size())); + + // Thread ID, Job ID, Name, Priority, Group (system, main), Status, % CPU + sb.append( + String.format( + "%10s %15s %-90s %5s %10s %-20s %-5s%n", + "Thread ID", + "Job ID", + "Name", + "Prio.", + "Group", + "Status", + "% CPU" + ) + ); + + for (NodeThreadInfo thread : threads) { + String line = String.format( + "%10s %15s %-90s %5s %10s %-20s %.2f%n", + thread.getId(), + thread.getJobId(), + thread.getName().substring(0, Math.min(90, thread.getName().length())), + thread.getPrio(), + thread.getGroupName().substring(0, Math.min(10, thread.getGroupName().length())), + thread.getState(), + thread.getCpuTime() / wallTime * 100 + ); + sb.append(line); + } + + return sb; + } + private int getPeerStatusCount(PeerNodeStatus[] peerNodeStatuses, int status) { int count = 0; for (PeerNodeStatus peerNodeStatus: peerNodeStatuses) { @@ -471,4 +527,4 @@ private String l10n(String key, String[] patterns, String[] values) { public String path() { return TOADLET_URL; } -} \ No newline at end of file +} diff --git a/src/freenet/clients/http/FileInsertWizardToadlet.java b/src/freenet/clients/http/FileInsertWizardToadlet.java index 1d28e1d3ddd..37beaeb2971 100644 --- a/src/freenet/clients/http/FileInsertWizardToadlet.java +++ b/src/freenet/clients/http/FileInsertWizardToadlet.java @@ -88,33 +88,42 @@ private HTMLNode createInsertBox (PageMaker pageMaker, ToadletContext ctx, boole NETWORK_THREAT_LEVEL seclevel = core.node.securityLevels.getNetworkThreatLevel(); HTMLNode insertForm = ctx.addFormChild(insertContent, QueueToadlet.PATH_UPLOADS, "queueInsertForm"); HTMLNode input = insertForm.addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "keytype", "CHK" }); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "keytype", "CHK", "keytypeChk" }); if ((!rememberedLastTime && seclevel == NETWORK_THREAT_LEVEL.LOW) || (rememberedLastTime && wasCanonicalLastTime && seclevel != NETWORK_THREAT_LEVEL.MAXIMUM)) { input.addAttribute("checked", "checked"); } - insertForm.addChild("b", l10n("insertCanonicalTitle")); + insertForm.addChild("label", + new String[] { "for" }, + new String[] { "keytypeChk" } + ).addChild("b", l10n("insertCanonicalTitle")); insertForm.addChild("#", ": "+l10n("insertCanonical")); if(isAdvancedModeEnabled) insertForm.addChild("#", " "+l10n("insertCanonicalAdvanced")); insertForm.addChild("br"); input = insertForm.addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "keytype", "SSK" }); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "keytype", "SSK", "keytypeSsk" }); if (seclevel == NETWORK_THREAT_LEVEL.MAXIMUM || (rememberedLastTime && !wasCanonicalLastTime)) { input.addAttribute("checked", "checked"); } - insertForm.addChild("b", l10n("insertRandomTitle")); + insertForm.addChild("label", + new String[] { "for" }, + new String[] { "keytypeSsk" } + ).addChild("b", l10n("insertRandomTitle")); insertForm.addChild("#", ": "+l10n("insertRandom")); if(isAdvancedModeEnabled) insertForm.addChild("#", " "+l10n("insertRandomAdvanced")); if (isAdvancedModeEnabled) { insertForm.addChild("br"); insertForm.addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "keytype", "specify" }); - insertForm.addChild("b", l10n("insertSpecificKeyTitle")); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "keytype", "specify", "keytypeSpecify" }); + insertForm.addChild("label", + new String[] { "for" }, + new String[] { "keytypeSpecify" } + ).addChild("b", l10n("insertSpecificKeyTitle")); insertForm.addChild("#", ": "+l10n("insertSpecificKey")+" "); insertForm.addChild("input", new String[] { "type", "name", "value" }, @@ -124,10 +133,12 @@ private HTMLNode createInsertBox (PageMaker pageMaker, ToadletContext ctx, boole insertForm.addChild("br"); insertForm.addChild("br"); insertForm.addChild("input", - new String[] { "type", "name", "checked" }, - new String[] { "checkbox", "compress", "checked" }); - insertForm.addChild("#", ' ' + - NodeL10n.getBase().getString("QueueToadlet.insertFileCompressLabel")); + new String[] { "type", "name", "checked", "id" }, + new String[] { "checkbox", "compress", "checked", "checkboxCompress" }); + insertForm.addChild("label", + new String[] { "for" }, + new String[] { "checkboxCompress" }, + ' ' + NodeL10n.getBase().getString("QueueToadlet.insertFileCompressLabel")); } else { insertForm.addChild("input", new String[] { "type", "name", "value" }, @@ -195,14 +206,20 @@ private HTMLNode createFilterBox (PageMaker pageMaker, ToadletContext ctx) { // display in browser or save to disk insertForm.addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "result-handling", ResultHandling.DISPLAY.toString() }); - insertForm.addChild("#", ContentFilterToadlet.l10n("displayResultLabel")); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "result-handling", ResultHandling.DISPLAY.toString(), "resHandlingDisplay" }); + insertForm.addChild("label", + new String[] { "for" }, + new String[] { "resHandlingDisplay" }, + ContentFilterToadlet.l10n("displayResultLabel")); insertForm.addChild("br"); insertForm.addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "result-handling", ResultHandling.SAVE.toString() }); - insertForm.addChild("#", ContentFilterToadlet.l10n("saveResultLabel")); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "result-handling", ResultHandling.SAVE.toString(), "resHandlingSave" }); + insertForm.addChild("label", + new String[] { "for" }, + new String[] { "resHandlingSave" }, + ContentFilterToadlet.l10n("saveResultLabel")); insertForm.addChild("br"); insertForm.addChild("br"); diff --git a/src/freenet/clients/http/PproxyToadlet.java b/src/freenet/clients/http/PproxyToadlet.java index 7318999735d..d413edcbf4b 100644 --- a/src/freenet/clients/http/PproxyToadlet.java +++ b/src/freenet/clients/http/PproxyToadlet.java @@ -569,17 +569,25 @@ private void showOfficialPluginLoader(ToadletContext toadletContext, HTMLNode co boolean loadFromWeb = pm.loadOfficialPluginsFromWeb(); - HTMLNode input = addOfficialForm.addChild("input", new String[] { "type", "name", "value" }, - new String[] { "radio", "pluginSource", "freenet" }); + HTMLNode input = addOfficialForm.addChild("input", + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "pluginSource", "freenet", "pluginSourceFreenet" }); if(!loadFromWeb) input.addAttribute("checked", "true"); - addOfficialForm.addChild("#", l10n("pluginSourceFreenet")); + addOfficialForm.addChild("label", + new String[] { "for" }, + new String[] { "pluginSourceFreenet" }, + l10n("pluginSourceFreenet")); addOfficialForm.addChild("br"); - input = addOfficialForm.addChild("input", new String[] { "type", "name", "value" }, - new String[] { "radio", "pluginSource", "https" }); + input = addOfficialForm.addChild("input", + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "pluginSource", "https", "pluginSourceHTTPS" }); if(loadFromWeb) input.addAttribute("checked", "true"); - addOfficialForm.addChild("#", l10n("pluginSourceHTTPS")); + addOfficialForm.addChild("label", + new String[] { "for" }, + new String[] { "pluginSourceHTTPS" }, + l10n("pluginSourceHTTPS")); addOfficialForm.addChild("#", " "); if(node.getOpennet() == null) addOfficialForm.addChild("b").addChild("font", "color", "red", l10n("pluginSourceHTTPSWarning")); @@ -602,8 +610,13 @@ private void showOfficialPluginLoader(ToadletContext toadletContext, HTMLNode co continue; } HTMLNode pluginNode = pluginGroupNode.addChild("div", "class", "plugin"); - HTMLNode option = pluginNode.addChild("input", new String[] { "type", "name", "value" }, new String[] { "radio", "plugin-name", pluginDescription.name }); - option.addChild("i", pluginDescription.getLocalisedPluginName()); + HTMLNode option = pluginNode.addChild("input", + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "plugin-name", pluginDescription.name, "radioPlugin" + pluginDescription.name }); + option.addChild("label", + new String[] { "for" }, + new String[] { "radioPlugin" + pluginDescription.name } + ).addChild("i", pluginDescription.getLocalisedPluginName()); if(pluginDescription.deprecated) option.addChild("b", " ("+l10n("loadLabelDeprecated")+")"); if(pluginDescription.experimental) @@ -639,8 +652,13 @@ private void showUnofficialPluginLoader(ToadletContext toadletContext, HTMLNode addOtherForm.addChild("#", " "); addOtherForm.addChild("input", new String[] { "type", "name", "value" }, new String[] { "submit", "submit-other", l10n("Load") }); addOtherForm.addChild("br"); - addOtherForm.addChild("input", new String[] { "type", "name", "checked" }, new String[] { "checkbox", "fileonly", "checked" }); - addOtherForm.addChild("#", " " + l10n("fileonly")); + addOtherForm.addChild("input", + new String[] { "type", "name", "checked", "id" }, + new String[] { "checkbox", "fileonly", "checked", "fileonly" }); + addOtherForm.addChild("label", + new String[] { "for" }, + new String[] { "fileonly" }, + " " + l10n("fileonly")); } private void showFreenetPluginLoader(ToadletContext toadletContext, HTMLNode contentNode) { diff --git a/src/freenet/clients/http/QueueToadlet.java b/src/freenet/clients/http/QueueToadlet.java index 1c78d7d2d77..f94dd5d76f9 100644 --- a/src/freenet/clients/http/QueueToadlet.java +++ b/src/freenet/clients/http/QueueToadlet.java @@ -2011,22 +2011,31 @@ private HTMLNode createBulkDownloadForm(ToadletContext ctx, PageMaker pageMaker) } else { downloadForm.addChild("br"); downloadForm.addChild("input", - new String[] { "type", "value", "name" }, - new String[] { "radio", "disk", "target" }, + new String[] { "type", "value", "name", "id" }, + new String[] { "radio", "disk", "target", "bulkDownloadSelectOptionDisk" } //Nicer spacing for radio button - ' '+l10n("bulkDownloadSelectOptionDisk")+' '); + ).addChild("label", + new String[] { "for" }, + new String[] { "bulkDownloadSelectOptionDisk" }, + ' '+l10n("bulkDownloadSelectOptionDisk")+' '); selectLocation(downloadForm); downloadForm.addChild("br"); downloadForm.addChild("input", - new String[] { "type", "value", "name", "checked" }, - new String[] { "radio", "direct", "target", "checked" }, - ' '+l10n("bulkDownloadSelectOptionDirect")+' '); + new String[] { "type", "value", "name", "checked", "id" }, + new String[] { "radio", "direct", "target", "checked", "bulkDownloadSelectOptionDirect" } + ).addChild("label", + new String[] { "for" }, + new String[] { "bulkDownloadSelectOptionDirect" }, + ' '+l10n("bulkDownloadSelectOptionDirect")+' '); } HTMLNode filterControl = downloadForm.addChild("div", l10n("filterData")); filterControl.addChild("input", - new String[] { "type", "name", "value", "checked" }, - new String[] { "checkbox", "filterData", "filterData", "checked"}); - filterControl.addChild("#", l10n("filterDataMessage")); + new String[] { "type", "name", "value", "checked", "id" }, + new String[] { "checkbox", "filterData", "filterData", "checked", "filterDataMessage"}); + filterControl.addChild("label", + new String[] { "for" }, + new String[] { "filterDataMessage" }, + l10n("filterDataMessage")); downloadForm.addChild("br"); downloadForm.addChild("input", new String[] { "type", "name", "value" }, diff --git a/src/freenet/clients/http/SecurityLevelsToadlet.java b/src/freenet/clients/http/SecurityLevelsToadlet.java index 81890fb8224..3f8d7f69651 100644 --- a/src/freenet/clients/http/SecurityLevelsToadlet.java +++ b/src/freenet/clients/http/SecurityLevelsToadlet.java @@ -491,12 +491,19 @@ private void drawSecurityLevelsPage(HTMLNode contentNode, ToadletContext ctx) { String controlName = "security-levels.networkThreatLevel"; for(NETWORK_THREAT_LEVEL level : NETWORK_THREAT_LEVEL.getOpennetValues()) { HTMLNode input; - if(level == networkLevel) { - input = div.addChild("p").addChild("input", new String[] { "type", "checked", "name", "value" }, new String[] { "radio", "on", controlName, level.name() }); + if (level == networkLevel) { + input = div.addChild("p").addChild("input", + new String[] { "type", "checked", "name", "value", "id" }, + new String[] { "radio", "on", controlName, level.name(), controlName + level.name() }); } else { - input = div.addChild("p").addChild("input", new String[] { "type", "name", "value" }, new String[] { "radio", controlName, level.name() }); + input = div.addChild("p").addChild("input", + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", controlName, level.name(), controlName + level.name() }); } - input.addChild("b", l10nSec("networkThreatLevel.name."+level)); + input.addChild("label", + new String[] { "for" }, + new String[] { controlName + level.name() } + ).addChild("b", l10nSec("networkThreatLevel.name."+level)); input.addChild("#", ": "); NodeL10n.getBase().addL10nSubstitution(input, "SecurityLevels.networkThreatLevel.choice."+level, new String[] { "bold" }, new HTMLNode[] { HTMLNode.STRONG }); @@ -514,11 +521,18 @@ private void drawSecurityLevelsPage(HTMLNode contentNode, ToadletContext ctx) { for(NETWORK_THREAT_LEVEL level : NETWORK_THREAT_LEVEL.getDarknetValues()) { HTMLNode input; if(level == networkLevel) { - input = div.addChild("p").addChild("input", new String[] { "type", "checked", "name", "value" }, new String[] { "radio", "on", controlName, level.name() }); + input = div.addChild("p").addChild("input", + new String[] { "type", "checked", "name", "value", "id" }, + new String[] { "radio", "on", controlName, level.name(), controlName + level.name() }); } else { - input = div.addChild("p").addChild("input", new String[] { "type", "name", "value" }, new String[] { "radio", controlName, level.name() }); + input = div.addChild("p").addChild("input", + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", controlName, level.name(), controlName + level.name() }); } - input.addChild("b", l10nSec("networkThreatLevel.name."+level)); + input.addChild("label", + new String[] { "for" }, + new String[] { controlName + level.name() } + ).addChild("b", l10nSec("networkThreatLevel.name."+level)); input.addChild("#", ": "); NodeL10n.getBase().addL10nSubstitution(input, "SecurityLevels.networkThreatLevel.choice."+level, new String[] { "bold" }, new HTMLNode[] { HTMLNode.STRONG }); @@ -552,12 +566,19 @@ private void drawSecurityLevelsPage(HTMLNode contentNode, ToadletContext ctx) { controlName = "security-levels.physicalThreatLevel"; for(PHYSICAL_THREAT_LEVEL level : PHYSICAL_THREAT_LEVEL.values()) { HTMLNode input; - if(level == physicalLevel) { - input = seclevelGroup.addChild("p").addChild("input", new String[] { "type", "checked", "name", "value" }, new String[] { "radio", "on", controlName, level.name() }); + if (level == physicalLevel) { + input = seclevelGroup.addChild("p").addChild("input", + new String[] { "type", "checked", "name", "value", "id" }, + new String[] { "radio", "on", controlName, level.name(), controlName + level.name() }); } else { - input = seclevelGroup.addChild("p").addChild("input", new String[] { "type", "name", "value" }, new String[] { "radio", controlName, level.name() }); + input = seclevelGroup.addChild("p").addChild("input", + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", controlName, level.name(), controlName + level.name() }); } - input.addChild("b", l10nSec("physicalThreatLevel.name."+level)); + input.addChild("label", + new String[] { "for" }, + new String[] { controlName + level.name() } + ).addChild("b", l10nSec("physicalThreatLevel.name."+level)); input.addChild("#", ": "); NodeL10n.getBase().addL10nSubstitution(input, "SecurityLevels.physicalThreatLevel.choice."+level, new String[] { "bold" }, new HTMLNode[] { HTMLNode.STRONG }); diff --git a/src/freenet/clients/http/complexhtmlnodes/PeerTrustInputForAddPeerBoxNode.java b/src/freenet/clients/http/complexhtmlnodes/PeerTrustInputForAddPeerBoxNode.java index 74d316c3627..659b0d9be45 100644 --- a/src/freenet/clients/http/complexhtmlnodes/PeerTrustInputForAddPeerBoxNode.java +++ b/src/freenet/clients/http/complexhtmlnodes/PeerTrustInputForAddPeerBoxNode.java @@ -15,11 +15,14 @@ public PeerTrustInputForAddPeerBoxNode() { for (DarknetPeerNode.FRIEND_TRUST trust : DarknetPeerNode.FRIEND_TRUST.valuesBackwards()) { // FIXME reverse order HTMLNode input = this.addChild("br") .addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "trust", trust.name() }); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "trust", trust.name(), "trust" + trust.name() }); if (trust.isDefaultValue()) input.addAttribute("checked", "checked"); - input.addChild("b", l10n("DarknetConnectionsToadlet.peerTrust." + trust.name())); + input.addChild("label", + new String[] { "for" }, + new String[] { "trust" + trust.name() } + ).addChild("b", l10n("DarknetConnectionsToadlet.peerTrust." + trust.name())); input.addChild("#", ": "); input.addChild("#", l10n("DarknetConnectionsToadlet.peerTrustExplain." + trust.name())); } diff --git a/src/freenet/clients/http/complexhtmlnodes/PeerVisibilityInputForAddPeerBoxNode.java b/src/freenet/clients/http/complexhtmlnodes/PeerVisibilityInputForAddPeerBoxNode.java index dc34337d068..45bee4f8a55 100644 --- a/src/freenet/clients/http/complexhtmlnodes/PeerVisibilityInputForAddPeerBoxNode.java +++ b/src/freenet/clients/http/complexhtmlnodes/PeerVisibilityInputForAddPeerBoxNode.java @@ -15,11 +15,14 @@ public PeerVisibilityInputForAddPeerBoxNode() { for (DarknetPeerNode.FRIEND_VISIBILITY visibility : DarknetPeerNode.FRIEND_VISIBILITY.values()) { // FIXME reverse order HTMLNode input = this.addChild("br") .addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "visibility", visibility.name() }); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "visibility", visibility.name(), "visibility" + visibility.name() }); if (visibility.isDefaultValue()) input.addAttribute("checked", "checked"); - input.addChild("b", l10n("DarknetConnectionsToadlet.peerVisibility." + visibility.name())); + input.addChild("label", + new String[] { "for" }, + new String[] { "visibility" + visibility.name() } + ).addChild("b", l10n("DarknetConnectionsToadlet.peerVisibility." + visibility.name())); input.addChild("#", ": "); input.addChild("#", l10n("DarknetConnectionsToadlet.peerVisibilityExplain." + visibility.name())); } diff --git a/src/freenet/clients/http/staticfiles/defaultbookmarks.dat b/src/freenet/clients/http/staticfiles/defaultbookmarks.dat index 1a3d2d71e2d..fe00429040b 100644 --- a/src/freenet/clients/http/staticfiles/defaultbookmarks.dat +++ b/src/freenet/clients/http/staticfiles/defaultbookmarks.dat @@ -27,7 +27,7 @@ BookmarkCategory0.Content.Bookmark3.URI=USK@z6lnOx7PPe16P3Sobu1KM9jiLiIxdvE3L918 BookmarkCategory1.Name=L10N:categoryDocumentation BookmarkCategory1.Content.BookmarkCategory=0 BookmarkCategory1.Content.Bookmark=8 -BookmarkCategory1.Content.Bookmark0.URI=USK@0iU87PXyodL2nm6kCpmYntsteViIbMwlJE~wlqIVvZ0,nenxGvjXDElX5RIZxMvwSnOtRzUKJYjoXEDgkhY6Ljw,AQACAAE/freenetproject-mirror/510/ +BookmarkCategory1.Content.Bookmark0.URI=USK@0iU87PXyodL2nm6kCpmYntsteViIbMwlJE~wlqIVvZ0,nenxGvjXDElX5RIZxMvwSnOtRzUKJYjoXEDgkhY6Ljw,AQACAAE/freenetproject-mirror/511/ BookmarkCategory1.Content.Bookmark0.Name=Freenetproject Website Mirror BookmarkCategory1.Content.Bookmark0.Description=L10N:freenetprojectorg BookmarkCategory1.Content.Bookmark0.ShortDescription=L10N:freenetprojectorg @@ -117,5 +117,5 @@ BookmarkCategory3.Content.Bookmark4.Name=ArneBab BookmarkCategory3.Content.Bookmark4.Description=L10N:Flog.ArneBab BookmarkCategory3.Content.Bookmark4.ShortDescription=L10N:Flog.ArneBab BookmarkCategory3.Content.Bookmark4.hasAnActivelink=true -BookmarkCategory3.Content.Bookmark4.URI=USK@sUm3oJISSEU4pl2Is9qa1eRoCLyz6r2LPkEqlXc3~oc,yBEbf-IJrcB8Pe~gAd53DEEHgbugUkFSHtzzLqnYlbs,AQACAAE/random_babcom/418/ +BookmarkCategory3.Content.Bookmark4.URI=USK@sUm3oJISSEU4pl2Is9qa1eRoCLyz6r2LPkEqlXc3~oc,yBEbf-IJrcB8Pe~gAd53DEEHgbugUkFSHtzzLqnYlbs,AQACAAE/random_babcom/424/ End diff --git a/src/freenet/clients/http/wizardsteps/DATASTORE_SIZE.java b/src/freenet/clients/http/wizardsteps/DATASTORE_SIZE.java index 068dd2da1ed..f8e5d0edc9c 100644 --- a/src/freenet/clients/http/wizardsteps/DATASTORE_SIZE.java +++ b/src/freenet/clients/http/wizardsteps/DATASTORE_SIZE.java @@ -44,8 +44,12 @@ public void getStep(HTTPRequest request, PageHelper helper) { @SuppressWarnings("unchecked") Option sizeOption = (Option) config.get("node").getOption("storeSize"); + @SuppressWarnings("unchecked") + Option clientCacheSizeOption = (Option) config.get("node").getOption("clientCacheSize"); + @SuppressWarnings("unchecked") + Option slashdotCacheSizeOption = (Option) config.get("node").getOption("slashdotCacheSize"); if(!sizeOption.isDefault()) { - long current = sizeOption.getValue(); + long current = sizeOption.getValue() + clientCacheSizeOption.getValue() + slashdotCacheSizeOption.getValue(); result.addChild("option", new String[] { "value", "selected" }, new String[] { SizeUtil.formatSize(current), "on" }, WizardL10n.l10n("currentPrefix")+" "+SizeUtil.formatSize(current)); diff --git a/src/freenet/clients/http/wizardsteps/MISC.java b/src/freenet/clients/http/wizardsteps/MISC.java index 32ea2e19eec..3c33da0f091 100644 --- a/src/freenet/clients/http/wizardsteps/MISC.java +++ b/src/freenet/clients/http/wizardsteps/MISC.java @@ -32,19 +32,28 @@ public void getStep(HTTPRequest request, PageHelper helper) { miscInfoboxContent.addChild("p", WizardL10n.l10n("autoUpdateLong")); miscInfoboxContent.addChild("p").addChild("input", - new String[] { "type", "checked", "name", "value" }, - new String[] { "radio", "on", "autodeploy", "true" }, WizardL10n.l10n("autoUpdateAutodeploy")); + new String[] { "type", "checked", "name", "value", "id" }, + new String[] { "radio", "on", "autodeploy", "true", "autodeployTrue" } + ).addChild("label", + new String[] { "for" }, + new String[] { "autodeployTrue" }, WizardL10n.l10n("autoUpdateAutodeploy")); miscInfoboxContent.addChild("p").addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "autodeploy", "false" }, WizardL10n.l10n("autoUpdateNoAutodeploy")); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "autodeploy", "false", "autodeployFalse" } + ).addChild("label", + new String[] { "for" }, + new String[] { "autodeployFalse" }, WizardL10n.l10n("autoUpdateNoAutodeploy")); miscInfoboxContent = helper.getInfobox("infobox-normal", WizardL10n.l10n("plugins"), form, null, false); miscInfoboxContent.addChild("p", WizardL10n.l10n("pluginsLong")); miscInfoboxContent.addChild("p").addChild("input", - new String[] { "type", "checked", "name", "value" }, - new String[] { "checkbox", "on", "upnp", "true" }, WizardL10n.l10n("enableUPnP")); + new String[] { "type", "checked", "name", "value", "id" }, + new String[] { "checkbox", "on", "upnp", "true", "upnpTrue" } + ).addChild("label", + new String[] { "for" }, + new String[] { "upnpTrue" }, WizardL10n.l10n("enableUPnP")); miscInfoboxContent.addChild("input", new String[] { "type", "name", "value" }, new String[] { "submit", "back", NodeL10n.getBase().getString("Toadlet.back")}); diff --git a/src/freenet/clients/http/wizardsteps/OPENNET.java b/src/freenet/clients/http/wizardsteps/OPENNET.java index bc97c26ec74..044dde985ad 100644 --- a/src/freenet/clients/http/wizardsteps/OPENNET.java +++ b/src/freenet/clients/http/wizardsteps/OPENNET.java @@ -22,9 +22,12 @@ public void getStep(HTTPRequest request, PageHelper helper) { HTMLNode p = form.addChild("p"); HTMLNode input = p.addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "opennet", "false" }); - input.addChild("b", WizardL10n.l10n("opennetChoiceConnectFriends")+":"); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "opennet", "false", "opennetFalse" }); + input.addChild("label", + new String[] { "for" }, + new String[] { "opennetFalse" } + ).addChild("b", WizardL10n.l10n("opennetChoiceConnectFriends")+":"); p.addChild("br"); p.addChild("i", WizardL10n.l10n("opennetChoicePro")); p.addChild("#", ": "+WizardL10n.l10n("opennetChoiceConnectFriendsPRO") + "ยน"); @@ -34,9 +37,12 @@ public void getStep(HTTPRequest request, PageHelper helper) { p = form.addChild("p"); input = p.addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "opennet", "true" }); - input.addChild("b", WizardL10n.l10n("opennetChoiceConnectStrangers")+":"); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "opennet", "true", "opennetTrue" }); + input.addChild("label", + new String[] { "for" }, + new String[] { "opennetTrue" } + ).addChild("b", WizardL10n.l10n("opennetChoiceConnectStrangers")+":"); p.addChild("br"); p.addChild("i", WizardL10n.l10n("opennetChoicePro")); p.addChild("#", ": "+WizardL10n.l10n("opennetChoiceConnectStrangersPRO")); diff --git a/src/freenet/clients/http/wizardsteps/SECURITY_NETWORK.java b/src/freenet/clients/http/wizardsteps/SECURITY_NETWORK.java index 1d02dd4909e..c8bb723d045 100644 --- a/src/freenet/clients/http/wizardsteps/SECURITY_NETWORK.java +++ b/src/freenet/clients/http/wizardsteps/SECURITY_NETWORK.java @@ -117,9 +117,12 @@ public void getStep(HTTPRequest request, PageHelper helper) { */ private void securityLevelChoice(HTMLNode parent, SecurityLevels.NETWORK_THREAT_LEVEL level) { HTMLNode input = parent.addChild("p").addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", "security-levels.networkThreatLevel", level.name() }); - input.addChild("b", WizardL10n.l10nSec("networkThreatLevel.name."+level)); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", "security-levels.networkThreatLevel", level.name(), "security-levels.networkThreatLevel" + level.name() }); + input.addChild("label", + new String[] { "for" }, + new String[] { "security-levels.networkThreatLevel" + level.name() } + ).addChild("b", WizardL10n.l10nSec("networkThreatLevel.name."+level)); input.addChild("#", ": "); NodeL10n.getBase().addL10nSubstitution(input, "SecurityLevels.networkThreatLevel.choice."+level, new String[] { "bold" }, diff --git a/src/freenet/clients/http/wizardsteps/SECURITY_PHYSICAL.java b/src/freenet/clients/http/wizardsteps/SECURITY_PHYSICAL.java index 625f7ad09c6..d1cfe660928 100644 --- a/src/freenet/clients/http/wizardsteps/SECURITY_PHYSICAL.java +++ b/src/freenet/clients/http/wizardsteps/SECURITY_PHYSICAL.java @@ -73,9 +73,12 @@ public void getStep(HTTPRequest request, PageHelper helper) { for(SecurityLevels.PHYSICAL_THREAT_LEVEL level : SecurityLevels.PHYSICAL_THREAT_LEVEL.values()) { HTMLNode input; input = div.addChild("p").addChild("input", - new String[] { "type", "name", "value" }, - new String[] { "radio", controlName, level.name() }); - input.addChild("b", WizardL10n.l10nSec("physicalThreatLevel.name." + level)); + new String[] { "type", "name", "value", "id" }, + new String[] { "radio", controlName, level.name(), controlName + level.name() }); + input.addChild("label", + new String[] { "for" }, + new String[] { controlName + level.name() } + ).addChild("b", WizardL10n.l10nSec("physicalThreatLevel.name." + level)); input.addChild("#", ": "); NodeL10n.getBase().addL10nSubstitution(input, "SecurityLevels.physicalThreatLevel.choice."+level, new String[] { "bold" }, new HTMLNode[] { HTMLNode.STRONG }); if(level == SecurityLevels.PHYSICAL_THREAT_LEVEL.HIGH && diff --git a/src/freenet/l10n/freenet.l10n.en.properties b/src/freenet/l10n/freenet.l10n.en.properties index 58fa08a7459..2e6ab831c1f 100644 --- a/src/freenet/l10n/freenet.l10n.en.properties +++ b/src/freenet/l10n/freenet.l10n.en.properties @@ -1105,6 +1105,8 @@ Node.enablePerNodeFailureTables=Enable per-node failure tables? Node.enablePerNodeFailureTablesLong=Enable automatically rerouting around nodes that failed a request within the last 10 minutes? Node.enableRoutedPing=Enable FNPRoutedPing? Node.enableRoutedPingLong=Enable FNPRoutedPing? Only useful in simulations, not on the real network. Turn it off. +Node.enableDiagnostics=Enable Diagnostics? +Node.enableDiagnosticsLong=By enabling Diagnostics the node will keep detailed information of its inner-workings (such as CPU usage per thread) which can help to troubleshoot problems. The collected data is kept in memory (i.e. not persisted on disk) and it's not sent to anybody over the network. Node.enableSwapping=Enable location swapping? Node.enableSwappingLong=Enable location swapping? Node.enableSwapQueueing=Enable queueing of swap requests? diff --git a/src/freenet/node/LocationManager.java b/src/freenet/node/LocationManager.java index 1a42ea59d0e..2a3f5f5be64 100644 --- a/src/freenet/node/LocationManager.java +++ b/src/freenet/node/LocationManager.java @@ -17,6 +17,7 @@ import java.nio.file.Files; import java.security.MessageDigest; import java.text.DateFormat; +import java.time.Clock; import java.time.Duration; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -69,6 +70,8 @@ public class LocationManager implements ByteCounter { public static final String FOIL_PITCH_BLACK_ATTACK_PREFIX = "mitigate-pitch-black-attack-"; + public static long PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY = DAYS.toMillis(1); + public static long PITCH_BLACK_MITIGATION_STARTUP_DELAY = MINUTES.toMillis(10); public class MyCallback extends SendMessageOnErrorCallback { @@ -117,6 +120,7 @@ public void acknowledged() { final SwapRequestSender sender; final Node node; long timeLastSuccessfullySwapped; + public static Clock clockForTesting = Clock.systemDefaultZone(); public LocationManager(RandomSource r, Node node) { loc = r.nextDouble(); @@ -189,18 +193,18 @@ public void run() { @Override public void run() { - node.ticker.queueTimedJob(this, DAYS.toMillis(1)); + node.ticker.queueTimedJob(this, PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY); if (swappingDisabled()) { return; } - LocalDateTime now = LocalDateTime.now(); + LocalDateTime now = LocalDateTime.now(clockForTesting); String isoDateStringToday = DateTimeFormatter.ISO_DATE .format(now); String isoDateStringYesterday = DateTimeFormatter.ISO_DATE - .format(now.minus(Duration.ofDays(1))); + .format(now.minusDays(1)); File[] previousInsertFromToday = node.userDir().dir() - .listFiles((file, name) -> name.startsWith(FOIL_PITCH_BLACK_ATTACK_PREFIX - + isoDateStringToday)); + .listFiles((file, name) -> name.startsWith(getPitchBlackPrefix( + isoDateStringToday))); HighLevelSimpleClient highLevelSimpleClient = node.clientCore.makeClient( RequestStarter.INTERACTIVE_PRIORITY_CLASS, true, @@ -211,13 +215,12 @@ public void run() { byte[] randomContentForKSK = new byte[20]; node.secureRandom.nextBytes(randomContentForKSK); String randomPart = Base64.encode(randomContentForKSK); - String nameForInsert = - FOIL_PITCH_BLACK_ATTACK_PREFIX + isoDateStringToday + "-" + randomPart; + String nameForInsert = getPitchBlackPrefix(isoDateStringToday + "-" + randomPart); tryToInsertPitchBlackCheck(highLevelSimpleClient, nameForInsert); } File[] foilPitchBlackStatusFiles = node.userDir().dir() - .listFiles((file, name) -> name.startsWith(FOIL_PITCH_BLACK_ATTACK_PREFIX)); + .listFiles((file, name) -> name.startsWith(getPitchBlackPrefix(""))); if (foilPitchBlackStatusFiles != null) { File[] successfulInsertFromYesterday = Arrays.stream(foilPitchBlackStatusFiles) .filter(file -> file.getName().contains(isoDateStringYesterday)) @@ -243,7 +246,11 @@ public void run() { } } } - }, MINUTES.toMillis(10)); + }, PITCH_BLACK_MITIGATION_STARTUP_DELAY); + } + + public String getPitchBlackPrefix(String middleSubstring) { + return FOIL_PITCH_BLACK_ATTACK_PREFIX + middleSubstring; } private void tryToRequestPitchBlackCheckFromYesterday( @@ -271,10 +278,14 @@ private void tryToRequestPitchBlackCheckFromYesterday( try { sskFetchResult = highLevelSimpleClient.fetch(insertFromYesterday.getURI()); if (!Arrays.equals(expectedContent, sskFetchResult.asByteArray())) { + // if we received false data, this is definitely an attack: move there to provide a good node in the location switchLocationToDefendAgainstPitchBlackAttack(insertFromYesterday); } } catch (FetchException e) { - if (isRequestExceptionBecauseUriIsNotAvailable(e)) { + if (isRequestExceptionBecauseUriIsNotAvailable(e) && node.fastWeakRandom.nextBoolean()) { + // switch to the attacked location with only 50% probability, + // because it could be caused by the defensive swap of another node + // which made its current content inaccessible. switchLocationToDefendAgainstPitchBlackAttack(insertFromYesterday); } return; @@ -303,7 +314,10 @@ private void tryToRequestPitchBlackCheckFromYesterday( try { highLevelSimpleClient.fetch(calculatedChkUri); } catch (FetchException e) { - if (isRequestExceptionBecauseUriIsNotAvailable(e)) { + if (isRequestExceptionBecauseUriIsNotAvailable(e) && node.fastWeakRandom.nextBoolean()) { + // switch to the attacked location with only 50% probability, + // because it could be caused by the defensive swap of another node + // which made its current content inaccessible. try { switchLocationToDefendAgainstPitchBlackAttack(new ClientCHK(calculatedChkUri)); } catch (MalformedURLException exception) { @@ -1592,6 +1606,14 @@ public Object[] getKnownLocations(long timestamp) { } } + public static void setClockForTesting(Clock clock) { + clockForTesting = clock; + } + + public static Clock getClockForTesting() { + return clockForTesting; + } + public static double[] extractLocs(PeerNode[] peers, boolean indicateBackoff) { double[] locs = new double[peers.length]; for(int i=0;i= -1) + throws InvalidConfigValueException { + if(val >= -1) { ResizablePersistentIntBuffer.setPersistenceTime(val); - else + storeSaltHashSlotFilterPersistenceTime = val; + } else throw new InvalidConfigValueException(l10n("slotFilterPersistenceTimeError")); } }, false); + storeSaltHashSlotFilterPersistenceTime = nodeConfig.getInt("storeSaltHashSlotFilterPersistenceTime"); nodeConfig.register("storeSaltHashResizeOnStart", false, sortOrder++, true, false, "Node.storeSaltHashResizeOnStart", "Node.storeSaltHashResizeOnStartLong", new BooleanCallback() { @@ -2536,7 +2544,38 @@ public void set(Boolean val) throws InvalidConfigValueException, }); enableRoutedPing = nodeConfig.getBoolean("enableRoutedPing"); - + + nodeConfig.register( + "enableNodeDiagnostics", + false, + sortOrder++, + true, + false, + "Node.enableDiagnostics", + "Node.enableDiagnosticsLong", + new BooleanCallback() { + @Override + public Boolean get() { + synchronized (Node.this) { + return enableNodeDiagnostics; + } + } + + @Override + public void set(Boolean val) { + synchronized (Node.this) { + enableNodeDiagnostics = val; + nodeDiagnostics.stop(); + + if (enableNodeDiagnostics) { + nodeDiagnostics.start(); + } + } + } + } + ); + enableNodeDiagnostics = nodeConfig.getBoolean("enableNodeDiagnostics"); + updateMTU(); // peers-offers/*.fref files @@ -2601,6 +2640,8 @@ public void realRun() { System.out.println("Node constructor completed"); new BandwidthManager(this).start(); + + nodeDiagnostics = new DefaultNodeDiagnostics(this.nodeStats, this.ticker); } private void peersOffersFrefFilesConfiguration(SubConfig nodeConfig, int configOptionSortOrder) { @@ -3166,6 +3207,10 @@ public void start(boolean noSwaps) throws NodeInitException { // Process any data in the extra peer data directory peers.readExtraPeerData(); + if (enableNodeDiagnostics) { + nodeDiagnostics.start(); + } + Logger.normal(this, "Started node"); hasStarted = true; @@ -4892,5 +4937,12 @@ public PluginManager getPluginManager() { DatabaseKey getDatabaseKey() { return databaseKey; } - + + public NodeDiagnostics getNodeDiagnostics() { + return nodeDiagnostics; + } + + public boolean isNodeDiagnosticsEnabled() { + return enableNodeDiagnostics; + } } diff --git a/src/freenet/node/Version.java b/src/freenet/node/Version.java index 421cf1a4bb1..84231641c01 100644 --- a/src/freenet/node/Version.java +++ b/src/freenet/node/Version.java @@ -50,7 +50,7 @@ public class Version { public static final String protocolVersion = "1.0"; /** The build number of the current revision */ - private static final int buildNumber = 1490; + private static final int buildNumber = 1491; /** Oldest build of fred we will talk to *before* _cal */ private static final int oldLastGoodBuild = 1474; diff --git a/src/freenet/node/diagnostics/DefaultNodeDiagnostics.java b/src/freenet/node/diagnostics/DefaultNodeDiagnostics.java new file mode 100644 index 00000000000..2f48012a668 --- /dev/null +++ b/src/freenet/node/diagnostics/DefaultNodeDiagnostics.java @@ -0,0 +1,46 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ +package freenet.node.diagnostics; + +import freenet.node.diagnostics.threads.*; +import freenet.support.Ticker; +import freenet.node.NodeStats; + +/** + * @author desyncr + * + * A class to retrieve data to build diagnostic dumps to help in determining + * node bottlenecks or misconfiguration. + * + * This class launches various threads at intervals to retrieve information. This information + * is available through the public methods. + * Some data pointers are obtained from NodeStats object. + */ +public class DefaultNodeDiagnostics implements NodeDiagnostics { + private final DefaultThreadDiagnostics defaultThreadDiagnostics; + + /** + * @param nodeStats Used to retrieve data points. + * @param ticker Used to queue timed jobs. + */ + public DefaultNodeDiagnostics(NodeStats nodeStats, Ticker ticker) { + defaultThreadDiagnostics = new DefaultThreadDiagnostics(nodeStats, ticker); + } + + public void start() { + defaultThreadDiagnostics.start(); + } + + public void stop() { + defaultThreadDiagnostics.stop(); + } + + /** + * @return List of threads registered in NodeStats.getThreads() + */ + @Override + public ThreadDiagnostics getThreadDiagnostics() { + return defaultThreadDiagnostics; + } +} diff --git a/src/freenet/node/diagnostics/NodeDiagnostics.java b/src/freenet/node/diagnostics/NodeDiagnostics.java new file mode 100644 index 00000000000..114099d31a4 --- /dev/null +++ b/src/freenet/node/diagnostics/NodeDiagnostics.java @@ -0,0 +1,10 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ +package freenet.node.diagnostics; + +import freenet.node.diagnostics.threads.*; + +public interface NodeDiagnostics { + ThreadDiagnostics getThreadDiagnostics(); +} diff --git a/src/freenet/node/diagnostics/ThreadDiagnostics.java b/src/freenet/node/diagnostics/ThreadDiagnostics.java new file mode 100644 index 00000000000..944d79ff92b --- /dev/null +++ b/src/freenet/node/diagnostics/ThreadDiagnostics.java @@ -0,0 +1,10 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ +package freenet.node.diagnostics; + +import freenet.node.diagnostics.threads.*; + +public interface ThreadDiagnostics { + NodeThreadSnapshot getThreadSnapshot(); +} diff --git a/src/freenet/node/diagnostics/threads/DefaultThreadDiagnostics.java b/src/freenet/node/diagnostics/threads/DefaultThreadDiagnostics.java new file mode 100644 index 00000000000..302b43bd98c --- /dev/null +++ b/src/freenet/node/diagnostics/threads/DefaultThreadDiagnostics.java @@ -0,0 +1,217 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ +package freenet.node.diagnostics.threads; + +import freenet.node.*; +import freenet.node.diagnostics.*; +import freenet.support.*; + +import java.io.*; +import java.lang.management.*; +import java.util.*; +import java.util.concurrent.atomic.*; +import java.util.stream.*; + +/** + * Runnable thread to retrieve node thread's information and compiling it into + * an array of NodeThreadInfo objects. + */ +public class DefaultThreadDiagnostics implements Runnable, ThreadDiagnostics { + private final String name; + private final int monitorInterval; + + private final NodeStats nodeStats; + private final Ticker ticker; + + /** Sleep interval to calculate % CPU used by each thread */ + private static final int DEFAULT_MONITOR_INTERVAL = 1000; + private static final String DEFAULT_MONITOR_THREAD_NAME = "NodeDiagnostics: thread monitor"; + + /** Initialising with an empty NodeThreadSnapshot to avoid possible race conditions */ + private final AtomicReference nodeThreadSnapshot = + new AtomicReference<>( + new NodeThreadSnapshot(new ArrayList<>(), DEFAULT_MONITOR_INTERVAL) + ); + + private final ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean(); + + /** Map to track thread's CPU differences between intervals of time */ + private final Map threadSnapshot = new HashMap<>(); + + /** + * @param nodeStats Used to retrieve data points + * @param ticker Used to queue timed jobs + * @param name Thread name + * @param monitorInterval Sleep intervals to retrieve CPU usage + */ + public DefaultThreadDiagnostics(NodeStats nodeStats, Ticker ticker, String name, int monitorInterval) { + this.nodeStats = nodeStats; + this.ticker = ticker; + this.name = name; + this.monitorInterval = monitorInterval; + } + + /** + * @param nodeStats Used to retrieve data points + * @param ticker Used to queue timed jobs + */ + public DefaultThreadDiagnostics(NodeStats nodeStats, Ticker ticker) { + this(nodeStats, ticker, DEFAULT_MONITOR_THREAD_NAME, DEFAULT_MONITOR_INTERVAL); + } + + /** + * @return Current snapshot. + */ + public NodeThreadSnapshot getThreadSnapshot() { + return nodeThreadSnapshot.get(); + } + + /** + * Schedule this class execution in seconds. + * + * @param interval Time internal in seconds. + */ + private void scheduleNext(int interval) { + ticker.queueTimedJob( + this, + name, + interval, + false, + true + ); + } + + /** + * Start the execution. + */ + public void start() { + scheduleNext(0); + } + + public void stop() { + ticker.removeQueuedJob(this); + } + + private void scheduleNext() { + scheduleNext(monitorInterval); + } + + /** + * Calculate the "delta" CPU time for a given thread. This method keeps + * track of the previous CPU Time and calculates the difference between that + * snapshot and the current CPU Time. + * + * If there's no previous snapshot of CPU Time for the thread this method + * will return 0. + * + * @param thread Thread object to get the CPU usage + * @return Delta CPU time (nanoseconds) + */ + private long getCpuTimeDelta(Thread thread) { + long jobId, current; + String name; + // Synchronizing thread to avoid PoolerExecutor to change thread name + // while we're measuring it. + synchronized (thread) { + name = thread.getName(); + current = threadMxBean.getThreadCpuTime(thread.getId()); + jobId = getJobId(thread); + } + + ThreadSnapshot snapshot = threadSnapshot.get(jobId); + long cpuUsage = current - (snapshot != null ? snapshot.getCpu() : 0); + threadSnapshot.put(jobId, new ThreadSnapshot(current, name)); + + return cpuUsage; + } + + /** + * Gets the job's ID from the thread (PooledExecutor.MyThread) or + * defaults to the thread's ID. + * @param thread + * @return Job ID or Thread ID. + */ + private long getJobId(Thread thread) { + long jobId = thread.getId(); + if ((thread instanceof PooledExecutor.MyThread)) { + jobId = ((PooledExecutor.MyThread) thread).getJobId(); + } + + return jobId; + } + + /** + * Class holder for cpu and thread name at the moment of measurement. This is + * necessary as the threads are pooled and may change name right after measurement. + */ + private static class ThreadSnapshot { + private final long cpu; + private final String name; + + public ThreadSnapshot(long cpu, String name) { + this.cpu = cpu; + this.name = name; + } + + public String getName() { + return name; + } + + public long getCpu() { + return cpu; + } + } + + + /** + * Remove threads that aren't present in the last snapshot. + * + * @param threads List of active threads. + */ + private void purgeInactiveThreads(List threads) { + List activeThreads = threads.stream() + .map(NodeThreadInfo::getJobId) // job id might be the same as thread id + .collect(Collectors.toList()); + + threadSnapshot.keySet() + .removeIf(key -> !activeThreads.contains(key)); + } + + /** + * Get thread (jobs) name at the time of measurement or default + * to current's thread name. + * @param thread + * @return Thread's name + */ + private String getJobName(Thread thread) { + ThreadSnapshot ts = threadSnapshot.get(getJobId(thread)); + return ts != null ? ts.getName() : thread.getName(); + } + + @Override + public void run() { + List threads = Arrays.stream(nodeStats.getThreads()) + .filter(Objects::nonNull) + .filter(thread -> thread.getThreadGroup() != null) + .filter(thread -> getJobId(thread) != 0) + .map(thread -> new NodeThreadInfo( + thread.getId(), + getJobId(thread), + getCpuTimeDelta(thread), + getJobName(thread), + thread.getPriority(), + thread.getThreadGroup().getName(), + thread.getState().toString() + ) + ) + .collect(Collectors.toList()); + + nodeThreadSnapshot.set( + new NodeThreadSnapshot(threads, monitorInterval) + ); + + purgeInactiveThreads(threads); + scheduleNext(); + } +} diff --git a/src/freenet/node/diagnostics/threads/NodeThreadInfo.java b/src/freenet/node/diagnostics/threads/NodeThreadInfo.java new file mode 100644 index 00000000000..bd06567272e --- /dev/null +++ b/src/freenet/node/diagnostics/threads/NodeThreadInfo.java @@ -0,0 +1,64 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ +package freenet.node.diagnostics.threads; + +/** + * Class to wrap node thread information. + */ +public class NodeThreadInfo { + private final long id; + private final long jobId; + private final String name; + private final int prio; + private final String groupName; + private final String state; + private final long cpuTime; + + /** + * @param id Thread ID + * @param jobId Job ID + * @param cpuTime Thread's CPU time in nanoseconds (delta) + * @param name Thread name, or + * @param prio Thread priority + * @param groupName Thread's group name + * @param state Thread current state (TIMED_WAITING, RUNNABLE, etc) + */ + NodeThreadInfo(long id, long jobId, long cpuTime, String name, int prio, String groupName, String state) { + this.id = id; + this.jobId = jobId; + this.name = name; + this.prio = prio; + this.groupName = groupName; + this.state = state; + this.cpuTime = cpuTime; + } + + public long getId() { + return id; + } + + public long getJobId() { + return jobId; + } + + public String getName() { + return name; + } + + public int getPrio() { + return prio; + } + + public String getGroupName() { + return groupName; + } + + public String getState() { + return state; + } + + public long getCpuTime() { + return cpuTime; + } +} diff --git a/src/freenet/node/diagnostics/threads/NodeThreadSnapshot.java b/src/freenet/node/diagnostics/threads/NodeThreadSnapshot.java new file mode 100644 index 00000000000..49158f27b43 --- /dev/null +++ b/src/freenet/node/diagnostics/threads/NodeThreadSnapshot.java @@ -0,0 +1,37 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ +package freenet.node.diagnostics.threads; + +import java.util.ArrayList; +import java.util.List; + +/** + * Wrapper class to contain a list of NodeThreadInfos. + */ +public class NodeThreadSnapshot { + private final List threads; + private final int interval; + + /** + * @param threads List of threads for this snapshot. + */ + public NodeThreadSnapshot(List threads, int interval) { + this.threads = new ArrayList<>(threads); + this.interval = interval; + } + + /** + * @return The snapshot's thread list. + */ + public List getThreads() { + return new ArrayList<>(threads); + } + + /** + * @return Snapshot interval. + */ + public int getInterval() { + return interval; + } +} diff --git a/src/freenet/node/simulator/RealNodePitchBlackMitigationTest.java b/src/freenet/node/simulator/RealNodePitchBlackMitigationTest.java new file mode 100644 index 00000000000..78cea15a447 --- /dev/null +++ b/src/freenet/node/simulator/RealNodePitchBlackMitigationTest.java @@ -0,0 +1,394 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ +package freenet.node.simulator; + +import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.MINUTES; + +import java.io.File; +import java.time.Clock; +import java.time.Duration; +import java.util.Arrays; +import java.util.stream.Collectors; + +import freenet.crypt.DummyRandomSource; +import freenet.crypt.RandomSource; +import freenet.node.LocationManager; +import freenet.node.Node; +import freenet.node.NodeStarter; +import freenet.node.PeerNode; +import freenet.support.Executor; +import freenet.support.Logger; +import freenet.support.Logger.LogLevel; +import freenet.support.PooledExecutor; +import freenet.support.io.FileUtil; +import freenet.support.math.BootstrappingDecayingRunningAverage; +import freenet.support.math.RunningAverage; +import freenet.support.math.SimpleRunningAverage; + +/** + * @author ArneBab + * + * This test spins uf Freenet nodes and simulates a pitch black attack and defense. + * It moves fake days forward to simulate the defense despite limited swapping speed. + * + * It spins up NUMBER_OF_NODES freenet nodes with DEEGREE. + * + * MIN_PINGS and MAX_PINGS give the minimum and maximum runtime. + * + * Adjust the variables NUMBER_OF_NODES, DEGREE, and PINGS_PER_ITERATION to adjust test parameters. + * + * Set PITCH_BLACK_ATTACK_MEAN_LOCATION and PITCH_BLACK_ATTACK_JITTER to select the location + * to attack. PITCH_BLACK_ATTACK_JITTER is necessary to prevent existind heuristics from deticting + * the naive attack with exactly one location. + * + * Set BETWEEN_PING_SLEEP_TIME to give the nodes time to swap between logging. + * + * PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY gives the time between triggering a mitigation + * (usually it is just one per day). It abvances the fake time by one day per period. + * + * Just grep the test output to get test results. Example Gnuplot calls to evaluate: + * + * set title "Average peer locations during pitch black mitigation" + * set xlabel "Time / Cycle" + * set ylabel "node / index" + * set cblabel "location / position in ring" + * plot "<(grep Cycle real-node-pitch-black-mitigation-test-results-11.log | grep ' node ' | sed 's/Cycle //;s/ node / /;s/: .*average=/ /;s/, .*$//;s/,/./g')" using 1:2:3 palette pt 5 ps 1.5 lw 1 title "RealNodePitchBlackMitigationTest" + * + * set title "Average path length of successful pings" + * set xlabel "Time / Cycle" + * set ylabel "average path lenth / hops" + * plot "<(grep 'Average path length' real-node-pitch-black-mitigation-test-results-11.log | sed 's/.*: //')" using 0:1 pt 5 ps 1.5 lw 1 title "RealNodePitchBlackMitigationTest" + * + * set title "Ping-Statistics" + * set xlabel "Time / Ping Number" + * set ylabel "fraction / unitless" + * set cblabel "path / hops needed" + * plot "<(grep 'Routed ping' real-node-pitch-black-mitigation-test-results-11.log | grep success | sed 's/Routed ping //;s/ success: / /g')" using 1:(($0+1)/$1):2 palette pt 3 ps 1 lw 1 title "succeeded", "<(grep 'Routed ping' real-node-pitch-black-mitigation-test-results-11.log | grep FAILED | sed 's/Routed ping //;s/FAILED from//')" using 1:(($0+1)/$1) pt 6 ps 1 lw 1 title "FAILED" + * + */ +public class RealNodePitchBlackMitigationTest extends RealNodeTest { + + static final int NUMBER_OF_NODES = 300; + static final int DEGREE = 4; + static final short MAX_HTL = (short) 10; + static final boolean START_WITH_IDEAL_LOCATIONS = true; + static final boolean FORCE_NEIGHBOUR_CONNECTIONS = true; + static final int MIN_PINGS = 420; + static final int MAX_PINGS = 840; + static final boolean ENABLE_SWAPPING = true; + static final boolean ENABLE_SWAP_QUEUEING = true; + static final boolean ENABLE_FOAF = true; + static final boolean ACTIVE_PITCH_BLACK_ATTACK = false; + static final boolean INITIAL_PITCH_BLACK_ATTACK = true; + public static final long PITCH_BLACK_MITIGATION_STARTUP_DELAY = MINUTES.toMillis(1); + public static final long PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY = MINUTES.toMillis(30); + public static final int PINGS_PER_ITERATION = 10; + + public static int DARKNET_PORT_BASE = RealNodeRequestInsertTest.DARKNET_PORT_END; + public static final int DARKNET_PORT_END = DARKNET_PORT_BASE + NUMBER_OF_NODES; + public static final double PITCH_BLACK_ATTACK_JITTER = 0.001; + public static final double PITCH_BLACK_ATTACK_MEAN_LOCATION = 0.5; + public static final int BETWEEN_PING_SLEEP_TIME = 500000; + + public static void main(String[] args) throws Exception { + System.out.println("Routing test using real nodes:"); + System.out.println(); + String dir = "realNodeRequestInsertTest"; + File wd = new File(dir); + if (!FileUtil.removeAll(wd)) { + System.err.println("Mass delete failed, test may not be accurate."); + System.exit(EXIT_CANNOT_DELETE_OLD_DATA); + } + wd.mkdir(); + //NOTE: globalTestInit returns in ignored random source + NodeStarter.globalTestInit(dir, false, LogLevel.ERROR, "", true); + // Make the network reproducible so we can easily compare different routing options by specifying a seed. + DummyRandomSource random = new DummyRandomSource(3142); + //DiffieHellman.init(random); + Node[] nodes = new Node[NUMBER_OF_NODES]; + Logger.normal(RealNodePitchBlackMitigationTest.class, "Creating nodes..."); + Executor executor = new PooledExecutor(); + for (int i = 0; i < NUMBER_OF_NODES; i++) { + System.err.println("Creating node " + i); + nodes[i] = NodeStarter.createTestNode( + DARKNET_PORT_BASE + i, + 0, + dir, + true, + MAX_HTL, + 0 /* no dropped packets */, + random, + executor, + 500 * NUMBER_OF_NODES, + 4000000, // 30 CHKs to avoid stray failures through overwriting + true, + ENABLE_SWAPPING, + false, + false, + false, + ENABLE_SWAP_QUEUEING, + true, + 0, + ENABLE_FOAF, + false, + true, + false, + null); + Logger.normal(RealNodePitchBlackMitigationTest.class, "Created node " + i); + } + Logger.normal(RealNodePitchBlackMitigationTest.class, "Created " + NUMBER_OF_NODES + " nodes"); + // Now link them up + makeKleinbergNetwork( + nodes, + START_WITH_IDEAL_LOCATIONS, + DEGREE, + FORCE_NEIGHBOUR_CONNECTIONS, + random); + + Logger.normal(RealNodePitchBlackMitigationTest.class, "Added random links"); + + // force a disrupted network + if (INITIAL_PITCH_BLACK_ATTACK) { + for (int i = 0; i < NUMBER_OF_NODES; i++) { + Node nodeToAttack = nodes[i]; + attackSpecificNode( + PITCH_BLACK_ATTACK_MEAN_LOCATION, + PITCH_BLACK_ATTACK_JITTER, + nodeToAttack, + i); + } + } + + // enable warning logging to see pitch black defense lo + Logger.globalSetThreshold(LogLevel.WARNING); + + // set the time to yesterday to have pitch black information + nodes[0].lm.setClockForTesting(Clock.offset(Clock.systemDefaultZone(), Duration.ofDays(-1))); + // shift forward one day per 5 minutes + Runnable dayIncrementingJob = new Runnable() { + @Override + public void run() { + nodes[0].ticker.queueTimedJob( + this, + PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY); + nodes[0].lm.setClockForTesting(Clock.offset( + nodes[0].lm.getClockForTesting(), + Duration.ofDays(1))); + } + }; + nodes[0].ticker.queueTimedJob( + dayIncrementingJob, + PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY); + + // start the nodes and adjust mitigation times + nodes[0].lm.PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY = PITCH_BLACK_MITIGATION_FREQUENCY_ONE_DAY; + nodes[0].lm.PITCH_BLACK_MITIGATION_STARTUP_DELAY = PITCH_BLACK_MITIGATION_STARTUP_DELAY; + for (int i = 0; i < NUMBER_OF_NODES; i++) { + System.err.println("Starting node " + i); + nodes[i].start(false); + } + + waitForAllConnected(nodes); + + // Make the choice of nodes to ping to and from deterministic too. + // There is timing noise because of all the nodes, but the network + // and the choice of nodes to start and finish are deterministic, so + // the overall result should be more or less deterministic. + waitForPingAverage(0.98, nodes, new DummyRandomSource(3143), MAX_PINGS, BETWEEN_PING_SLEEP_TIME); + System.exit(0); + } + + public static void attackSpecificNode( + double pitchBlackAttackMeanLocation, + double pitchBlackAttackJitter, + Node nodeToAttack, + int indexOfNode) { + double pitchBlackFakeLocation = pitchBlackAttackMeanLocation + + (nodeToAttack.fastWeakRandom.nextDouble() * pitchBlackAttackJitter); + System.err.println("Pitch-Black-Attack on node " + + indexOfNode + + " using mean " + + pitchBlackAttackMeanLocation + + " with jitter " + + pitchBlackAttackJitter + + ": " + + pitchBlackFakeLocation); + nodeToAttack.setLocation(pitchBlackFakeLocation); + System.err.println("New location of node " + indexOfNode + ": " + nodeToAttack.getLocation()); + } + + static void waitForPingAverage( + double accuracy, + Node[] nodes, + RandomSource random, + int maxTests, + int sleepTime) throws InterruptedException { + int totalHopsTaken = 0; + int cycleNumber = 0; + int lastSwaps = 0; + int lastNoSwaps = 0; + int failures = 0; + int successes = 0; + RunningAverage avg = new SimpleRunningAverage(100, 0.0); + RunningAverage avg2 = new BootstrappingDecayingRunningAverage(0.0, 0.0, 1.0, 100, null); + int pings = 0; + for (int total = 0; total < maxTests; total++) { + cycleNumber++; + if (ACTIVE_PITCH_BLACK_ATTACK) { + for (int i = 0; i < NUMBER_OF_NODES; i++) { + Node nodeToAttack = nodes[i]; + // attack 2% of the nodes per round + if (nodeToAttack.fastWeakRandom.nextFloat() < 0.98) { + continue; + } + attackSpecificNode( + PITCH_BLACK_ATTACK_MEAN_LOCATION, + PITCH_BLACK_ATTACK_JITTER, + nodeToAttack, + i); + } + } + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + // Ignore + } + for (int i = 0; i < nodes.length; i++) { + System.err.println("Cycle " + cycleNumber + " node " + i + ": " + nodes[i].getLocation() + " degree: " + nodes[i].getPeerNodes().length + " locs: " + Arrays + .stream(nodes[i].getPeerNodes()).map(PeerNode::getLocation).collect(Collectors.summarizingDouble(d -> d))); + } + int newSwaps = LocationManager.swaps; + int totalStarted = LocationManager.startedSwaps; + int noSwaps = LocationManager.noSwaps; + System.err.println("Swaps: " + (newSwaps - lastSwaps)); + System.err.println("\nTotal swaps: Started*2: " + + totalStarted * 2 + + ", succeeded: " + + newSwaps + + ", last minute failures: " + + noSwaps + + + ", ratio " + + (double) noSwaps / (double) newSwaps + + ", early failures: " + + ((totalStarted * 2) - (noSwaps + newSwaps))); + System.err.println("This cycle ratio: " + ((double) (noSwaps - lastNoSwaps)) / ((double) ( + newSwaps + - lastSwaps))); + lastNoSwaps = noSwaps; + System.err.println("Swaps rejected (already locked): " + + LocationManager.swapsRejectedAlreadyLocked); + System.err.println("Swaps rejected (nowhere to go): " + + LocationManager.swapsRejectedNowhereToGo); + System.err.println("Swaps rejected (rate limit): " + LocationManager.swapsRejectedRateLimit); + System.err.println("Swaps rejected (recognized ID):" + + LocationManager.swapsRejectedRecognizedID); + System.err.println("Swaps failed:" + LocationManager.noSwaps); + System.err.println("Swaps succeeded:" + LocationManager.swaps); + + double totalSwapInterval = 0.0; + double totalSwapTime = 0.0; + for (int i = 0; i < nodes.length; i++) { + totalSwapInterval += nodes[i].lm.getSendSwapInterval(); + totalSwapTime += nodes[i].lm.getAverageSwapTime(); + } + System.err.println("Average swap time: " + (totalSwapTime / nodes.length)); + System.err.println("Average swap sender interval: " + (totalSwapInterval / nodes.length)); + + waitForAllConnected(nodes); + + lastSwaps = newSwaps; + // Do some (routed) test-pings + for (int i = 0; i < PINGS_PER_ITERATION; i++) { + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e1) { + } + try { + Node randomNode = nodes[random.nextInt(nodes.length)]; + Node randomNode2 = randomNode; + while (randomNode2 == randomNode) { + randomNode2 = nodes[random.nextInt(nodes.length)]; + } + double loc2 = randomNode2.getLocation(); + Logger.normal( + RealNodePitchBlackMitigationTest.class, + "Pinging " + randomNode2.getDarknetPortNumber() + " @ " + loc2 + " from " + randomNode + .getDarknetPortNumber() + " @ " + randomNode.getLocation()); + + int hopsTaken = randomNode.routedPing(loc2, randomNode2.getDarknetPubKeyHash()); + pings++; + if (hopsTaken < 0) { + failures++; + avg.report(0.0); + avg2.report(0.0); + double ratio = (double) successes / ((double) (failures + successes)); + System.err.println("Routed ping " + + pings + + " FAILED from " + + randomNode.getDarknetPortNumber() + + " to " + + randomNode2.getDarknetPortNumber() + + " (long:" + + ratio + + ", short:" + + avg.currentValue() + + ", vague:" + + avg2.currentValue() + + ')'); + } else { + totalHopsTaken += hopsTaken; + successes++; + avg.report(1.0); + avg2.report(1.0); + double ratio = (double) successes / ((double) (failures + successes)); + System.err.println("Routed ping " + + pings + + " success: " + + hopsTaken + + ' ' + + randomNode.getDarknetPortNumber() + + " to " + + randomNode2.getDarknetPortNumber() + + " (long:" + + ratio + + ", short:" + + avg.currentValue() + + ", vague:" + + avg2.currentValue() + + ')'); + } + } catch (Throwable t) { + Logger.error(RealNodePitchBlackMitigationTest.class, "Caught " + t, t); + } + } + System.err.println("Average path length for successful requests: " + + ((double) totalHopsTaken) / successes); + if (pings > MAX_PINGS || pings > MIN_PINGS && avg.currentValue() > accuracy && ((double) successes / ((double) (failures + + successes)) > accuracy)) { + System.err.println(); + System.err.println("Reached " + (accuracy * 100) + "% accuracy."); + System.err.println(); + System.err.println("Network size: " + nodes.length); + System.err.println("Maximum HTL: " + MAX_HTL); + System.err.println("Average path length for successful requests: " + + totalHopsTaken / successes); + System.err.println("Total started swaps: " + LocationManager.startedSwaps); + System.err.println("Total rejected swaps (already locked): " + + LocationManager.swapsRejectedAlreadyLocked); + System.err.println("Total swaps rejected (nowhere to go): " + + LocationManager.swapsRejectedNowhereToGo); + System.err.println("Total swaps rejected (rate limit): " + + LocationManager.swapsRejectedRateLimit); + System.err.println("Total swaps rejected (recognized ID):" + + LocationManager.swapsRejectedRecognizedID); + System.err.println("Total swaps failed:" + LocationManager.noSwaps); + System.err.println("Total swaps succeeded:" + LocationManager.swaps); + return; + } + } + System.exit(EXIT_PING_TARGET_NOT_REACHED); + } +} diff --git a/src/freenet/node/simulator/readme.txt b/src/freenet/node/simulator/readme.txt index 8f11ab21325..614c373b63c 100644 --- a/src/freenet/node/simulator/readme.txt +++ b/src/freenet/node/simulator/readme.txt @@ -3,3 +3,5 @@ To run simulations, do something like: java -cp freenet.jar:freenet-ext.jar freenet.node.simulator.RealNodeProbeTest On Windows the classpath separator is ; instead of :. + +To test the pitch black mitigation, see RealNodePitchBlackMitigationTest.java diff --git a/src/freenet/node/updater/UpdateDeployContext.java b/src/freenet/node/updater/UpdateDeployContext.java index 7ba8799a230..b478128fce1 100644 --- a/src/freenet/node/updater/UpdateDeployContext.java +++ b/src/freenet/node/updater/UpdateDeployContext.java @@ -20,6 +20,7 @@ import freenet.node.NodeStarter; import freenet.node.updater.MainJarDependenciesChecker.Dependency; import freenet.node.updater.MainJarDependenciesChecker.MainJarDependencies; +import freenet.support.JVMVersion; import freenet.support.Logger; import freenet.support.io.Closer; @@ -146,7 +147,9 @@ void rewriteWrapperConf(boolean writtenNewJar) throws IOException, UpdateCatastr boolean writtenAnchorInterval = false; /** Add the relative JNA tempdir if it does not exist already */ boolean writtenJnaTmpDir = false; - + /** Allow accessing internal modules in Java 16+ */ + boolean writtenIllegalAccessPermit = false; + String newMain = mainJarAbsolute ? newMainJar.getAbsolutePath() : newMainJar.getPath(); String mainRHS = null; @@ -177,7 +180,7 @@ void rewriteWrapperConf(boolean writtenNewJar) throws IOException, UpdateCatastr // Ignore the numbers. String rhs = line.substring(idx+1); dontWrite = true; - if(rhs.equals("freenet.jar") || rhs.equals("freenet.jar.new") || + if(rhs.equals("freenet.jar") || rhs.equals("freenet.jar.new") || rhs.equals("freenet-stable-latest.jar") || rhs.equals("freenet-stable-latest.jar.new") || rhs.equals("freenet-testing-latest.jar") || rhs.equals("freenet-testing-latest.jar.new")) { if(writtenNewJar) @@ -211,6 +214,9 @@ void rewriteWrapperConf(boolean writtenNewJar) throws IOException, UpdateCatastr if (rhs.startsWith("-Djava.io.tmpdir=")) { writtenJnaTmpDir = true; } + if (rhs.startsWith("--illegal-access=permit")) { + writtenIllegalAccessPermit = true; + } } } else if(lowcaseLine.equals("wrapper.restart.reload_configuration=true")) { writtenReload = true; @@ -258,7 +264,12 @@ void rewriteWrapperConf(boolean writtenNewJar) throws IOException, UpdateCatastr if (!writtenJnaTmpDir) { bw.write("wrapper.java.additional."+count+"=-Djava.io.tmpdir=./tmp/"+'\n'); } - + + // allow accessing internal modules (required for Java 16+, only supported since Java 9) + if (!writtenIllegalAccessPermit && JVMVersion.supportsModules()) { + bw.write("wrapper.java.additional."+count+"=--illegal-access=permit"+'\n'); + } + for(String s : otherLines) bw.write(s+'\n'); diff --git a/src/freenet/support/JVMVersion.java b/src/freenet/support/JVMVersion.java index a40fed70840..8674ab9edeb 100644 --- a/src/freenet/support/JVMVersion.java +++ b/src/freenet/support/JVMVersion.java @@ -24,6 +24,11 @@ public class JVMVersion { */ public static final String UPDATER_THRESHOLD = "1.8"; + /** + * Oldest Java version which supports Modules + */ + public static final String SUPPORTS_MODULES_THRESHOLD = "1.9"; + /** * Pre-9 is formatted as: major.feature[.maintenance[_update]]-ident * Post-9 is formatted as: major[.minor[.security[. ...]]]-ident @@ -72,6 +77,15 @@ public static final boolean is32Bit() { } } + public static final boolean supportsModules() { + String currentVersion = getCurrent(); + if (currentVersion == null) { + return false; + } + + return compareVersion(SUPPORTS_MODULES_THRESHOLD, currentVersion) <= 0; + } + /** * Decomposes a version string into major, feature, and optional maintenance and update * components. diff --git a/src/freenet/support/PooledExecutor.java b/src/freenet/support/PooledExecutor.java index 4891136b245..cf9e2dd0f3c 100644 --- a/src/freenet/support/PooledExecutor.java +++ b/src/freenet/support/PooledExecutor.java @@ -6,6 +6,7 @@ import static java.util.concurrent.TimeUnit.MINUTES; import java.util.ArrayList; +import java.util.Random; import java.util.concurrent.atomic.AtomicLong; import freenet.node.PrioRunnable; @@ -161,17 +162,24 @@ public int getWaitingThreadsCount() { private static class Job { private final Runnable runnable; private final String name; + private final int id; Job(Runnable runnable, String name) { this.runnable = runnable; this.name = name; + this.id = new Random().nextInt(); + } + + public int getId() { + return id; } } - private class MyThread extends NativeThread { + public class MyThread extends NativeThread { final String defaultName; volatile boolean alive = true; Job nextJob; + Job job; final long threadNo; private boolean removed = false; @@ -195,12 +203,14 @@ public void realRun() { } } } + + public int getJobId() { + return job != null ? job.id : nextJob != null ? nextJob.id : 0; + } private void innerRun(int nativePriority) { long ranJobs = 0; while(true) { - Job job; - synchronized(this) { job = nextJob; nextJob = null; diff --git a/src/freenet/support/io/SkipShieldingInputStream.java b/src/freenet/support/io/SkipShieldingInputStream.java index e63b67331ff..bc8833723d3 100644 --- a/src/freenet/support/io/SkipShieldingInputStream.java +++ b/src/freenet/support/io/SkipShieldingInputStream.java @@ -46,6 +46,16 @@ public SkipShieldingInputStream(InputStream in) { @Override public long skip(long n) throws IOException { - return n < 0 ? 0 : read(SKIP_BUFFER, 0, (int) Math.min(n, SKIP_BUFFER_SIZE)); + int retval; + if (n < 0) { + retval = 0; + } + else { + retval = read(SKIP_BUFFER, 0, (int) Math.min(n, SKIP_BUFFER_SIZE)); + if (retval < 0) { + retval = 0; + } + } + return retval; } }