diff --git a/lib/internal/worker.js b/lib/internal/worker.js
index 26c16f86e3e8fc..a1abcff567d424 100644
--- a/lib/internal/worker.js
+++ b/lib/internal/worker.js
@@ -408,7 +408,6 @@ function setupChild(evalScript) {
     if (message.type === messageTypes.LOAD_SCRIPT) {
       const { filename, doEval, workerData, publicPort, hasStdin } = message;
       publicWorker.parentPort = publicPort;
-      setupPortReferencing(publicPort, publicPort, 'message');
       publicWorker.workerData = workerData;
 
       if (!hasStdin)
diff --git a/test/parallel/test-worker-parent-port-ref.js b/test/parallel/test-worker-parent-port-ref.js
new file mode 100644
index 00000000000000..48963f320f4450
--- /dev/null
+++ b/test/parallel/test-worker-parent-port-ref.js
@@ -0,0 +1,25 @@
+// Flags: --experimental-worker
+'use strict';
+const assert = require('assert');
+const common = require('../common');
+const { isMainThread, parentPort, Worker } = require('worker_threads');
+
+// This test makes sure that we manipulate the references of
+// `parentPort` correctly so that any worker threads will
+// automatically exit when there are no any other references.
+{
+  if (isMainThread) {
+    const worker = new Worker(__filename);
+
+    worker.on('exit', common.mustCall((code) => {
+      assert.strictEqual(code, 0);
+    }), 1);
+
+    worker.on('online', common.mustCall());
+  } else {
+    const messageCallback = () => {};
+    parentPort.on('message', messageCallback);
+    // The thread won't exit if we don't make the 'message' listener off.
+    parentPort.off('message', messageCallback);
+  }
+}