diff --git a/.travis.yml b/.travis.yml
index 20b64eb3..568aa88c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,7 +27,8 @@ install:
   - $PKG_CMD install
 script:
   - 'if [ -n "${LINT-}" ]; then $PKG_CMD run lint ; fi'
-  - 'if [ -z "${LINT-}" ]; then $PKG_CMD run test-ci ; fi'
+  - 'if [ -n "${SMOKE_TEST-}" ]; then node scripts/smoke-test.js ; fi'
+  - 'if [ -z "${LINT-}" ] && [ -z "${SMOKE_TEST-}" ]; then $PKG_CMD run test-ci ; fi'
 matrix:
   fast_finish: true
   exclude:
@@ -38,5 +39,7 @@ matrix:
   include:
     - node_js: "node"
       env: LINT=true PKG_CMD="npm"
+    - node_js: "node"
+      env: SMOKE_TEST=true PKG_CMD="npm"
 
-after_success: 'if [ -z "${LINT-}" ]; then npm run coverage-ci ; fi'
+after_success: 'if [ -z "${LINT-}" ] && [ -z "${SMOKE_TEST-}" ]; then npm run coverage-ci ; fi'
diff --git a/scripts/smoke-test.js b/scripts/smoke-test.js
new file mode 100755
index 00000000..3c81d8f2
--- /dev/null
+++ b/scripts/smoke-test.js
@@ -0,0 +1,44 @@
+const fs = require("fs-extra");
+const execSync = require("child_process").execSync;
+const path = require("path");
+const pkg = require("../package.json");
+
+let errorOccurred = false;
+
+const tempFolderPath = path.join(__dirname, "../tmp");
+const packPath = path.join(__dirname, `../babel-preset-env-${pkg.version}.tgz`);
+
+try {
+  console.log("Creating package");
+  execSync("npm pack");
+
+  console.log("Setting up smoke test");
+  fs.ensureDir(tempFolderPath);
+
+  fs.writeFileSync(
+    path.join(tempFolderPath, "package.json"),
+    "{\n  \"name\": \"babel-preset-env-smoke-test\",\n  \"version\": \"1.0.0\",\n  \"private\": true\n}\n"
+  );
+
+  fs.writeFileSync(
+    path.join(tempFolderPath, "index.js"),
+    "require(\"babel-preset-env\");\n"
+  );
+
+  process.chdir(tempFolderPath);
+
+  console.log("Running smoke test");
+  execSync(`npm install ../babel-preset-env-${pkg.version}.tgz`);
+  execSync("node index.js");
+} catch (e) {
+  errorOccurred = true;
+}
+
+console.log("Cleaning up");
+if (fs.existsSync(tempFolderPath)) {
+  fs.removeSync(tempFolderPath);
+}
+
+fs.removeSync(packPath);
+
+process.exit(errorOccurred ? 1 : 0);