Merge branch 'master' into gh-6658

docs/documents.pug

@@ -71,7 +71,7 @@ block content
     ```javascript
     doc.name = 'foo';
 
-    // Mongoose sends a `updateOne({ _id: doc._id }, { $set: { name: 'foo' } })`
+    // Mongoose sends an `updateOne({ _id: doc._id }, { $set: { name: 'foo' } })`
     // to MongoDB.
     await doc.save();
     ```
@@ -108,7 +108,7 @@ block content
 
     ### Validating
 
-    Documents are casted validated before they are saved. Mongoose first casts
+    Documents are casted and validated before they are saved. Mongoose first casts
     values to the specified type and then validates them. Internally, Mongoose
     calls the document's [`validate()` method](api.html#document_Document-validate)
     before saving.

docs/faq.pug

@@ -30,6 +30,7 @@ block content
       <a href="#native_link#"><span class="sponsor">Sponsor</span> #native_company# — #native_desc#</a>
     </div>
 
+
     <hr id="array-changes-not-saved" />
 
     <a href="#array-changes-not-saved">**Q**</a>. Why don't my changes to arrays get saved when I update an element
@@ -72,6 +73,7 @@ block content
     doc.save();
     ```
 
+
     <hr id="unique-doesnt-work" />
 
     <a href="#unique-doesnt-work">**Q**</a>. I declared a schema property as `unique` but I can still save
@@ -124,6 +126,7 @@ block content
     rather than relying on mongoose to do it for you. The `unique` option for schemas is
     convenient for development and documentation, but mongoose is *not* an index management solution.
 
+
     <hr id="nested-properties" />
 
     <a href="#nested-properties">**Q**</a>. When I have a nested property in a schema, mongoose adds empty objects by default. Why?
@@ -154,6 +157,7 @@ block content
     must always be defined as an object on a mongoose document, even if `nested`
     is undefined on the underlying [POJO](./guide.html#minimize).
 
+
     <hr id="destructured-imports" />
 
     <a href="#destructured-imports">**Q**</a>. When I use named imports like `import { set } from 'mongoose'`, I
@@ -179,6 +183,7 @@ block content
     foo(); // "undefined"
     ```
 
+
     <hr id="arrow-functions" />
 
     <a href="#arrow-functions">**Q**</a>. I'm using an arrow function for a [virtual](./guide.html#virtuals), [middleware](./middleware.html), [getter](./api.html#schematype_SchemaType-get)/[setter](./api.html#schematype_SchemaType-set), or [method](./guide.html#methods) and the value of `this` is wrong.
@@ -209,6 +214,7 @@ block content
     });
     ```
 
+
     <hr id="type-key">
 
     <a href="#type-key">**Q**</a>. I have an embedded property named `type` like this:
@@ -253,6 +259,7 @@ block content
     });
     ```
 
+
     <hr id="populate_sort_order" />
 
     <a href="#populate_sort_order">**Q**</a>. I'm populating a nested property under an array like the below code:
@@ -278,6 +285,7 @@ block content
     connect to MongoDB. Read the [buffering section of the connection docs](./connections.html#buffering)
     for more information.
 
+
     <hr id="enable_debugging" />
 
     <a href="#enable_debugging">**Q**</a>. How can I enable debugging?
@@ -294,6 +302,7 @@ block content
     All executed collection methods will log output of their arguments to your
     console.
 
+
     <hr id="callback_never_executes" />
 
     <a href="#callback_never_executes">**Q**</a>. My `save()` callback never executes. What am I doing wrong?
@@ -319,13 +328,15 @@ block content
     mongoose.set('bufferCommands', false);
     ```
 
+
     <hr id="creating_connections" />
 
     <a href="#creating_connections">**Q**</a>. Should I create/destroy a new connection for each database operation?
 
     **A**. No. Open your connection when your application starts up and leave
     it open until the application shuts down.
 
+
     <hr id="overwrite-model-error" />
 
     <a href="#overwrite-model-error">**Q**</a>. Why do I get "OverwriteModelError: Cannot overwrite .. model once
@@ -351,6 +362,7 @@ block content
     var Kitten = connection.model('Kitten', kittySchema);
     ```
 
+
     <hr id="array-defaults" />
 
     <a href="#array-defaults">**Q**</a>. How can I change mongoose's default behavior of initializing an array
@@ -365,6 +377,8 @@ block content
       }
     });
     ```
+
+
     <hr id="initialize-array-path-null" />
 
     <a href="#initialize-array-path-null">**Q**</a>. How can I initialize an array path to `null`?
@@ -389,6 +403,7 @@ block content
     to query by date using the aggregation framework, you're responsible for ensuring
     that you're passing in a valid date.
 
+
     <hr id="date_changes" />
 
     <a href="#date_changes">**Q**</a>. Why don't in-place modifications to date objects
@@ -429,19 +444,6 @@ block content
     **A**. Technically, any 12 character string is a valid [ObjectId](https://docs.mongodb.com/manual/reference/bson-types/#objectid).
     Consider using a regex like `/^[a-f0-9]{24}$/` to test whether a string is exactly 24 hex characters.
 
-    <hr id="slow-localhost" />
-
-    <a href="#slow-localhost">**Q**</a>. I'm connecting to `localhost` and it takes me nearly 1 second to connect. How do I fix this?
-
-    **A**. The underlying MongoDB driver defaults to looking for IPv6 addresses, so the most likely cause is that your `localhost` DNS mapping isn't configured to handle IPv6. Use `127.0.0.1` instead of `localhost` or use the `family` option as shown in the [connection docs](https://mongoosejs.com/docs/connections.html#options).
-
-    ```javascript
-    // One alternative is to bypass 'localhost'
-    mongoose.connect('mongodb://127.0.0.1:27017/test');
-    // Another option is to specify the `family` option, which tells the
-    // MongoDB driver to only look for IPv4 addresses rather than IPv6 first.
-    mongoose.connect('mongodb://localhost:27017/test', { family: 4 });
-    ```
 
     <hr id="map-keys-strings" />
 
@@ -458,6 +460,7 @@ block content
     [perDocumentLimit](/docs/populate.html#limit-vs-perDocumentLimit) option (new in Mongoose 5.9.0). Just keep in
     mind that populate() will execute a separate query for each document.
 
+
     <hr id="add_something" />
 
     **Something to add?**

docs/guide.pug

@@ -688,7 +688,7 @@ block content
     _Note that Mongoose does not send the `shardcollection` command for you. You
     must configure your shards yourself._
 
-    <h3 id="strict">option: strict</h3>
+    <h3 id="strict"><a href="#strict">option: strict</a></h3>
 
     The strict option, (enabled by default), ensures that values passed to our
     model constructor that were not specified in our schema do not get saved to
@@ -739,7 +739,7 @@ block content
     thing.save(); // iAmNotInTheSchema is never saved to the db
     ```
 
-    <h3 id="strictQuery">option: strictQuery</h3>
+    <h3 id="strictQuery"><a href="#strictQuery">option: strictQuery</a></h3>
 
     For backwards compatibility, the `strict` option does **not** apply to
     the `filter` parameter for queries.

docs/schematypes.pug

@@ -348,7 +348,7 @@ block content
     The values `null` and `undefined` are not cast.
 
     NaN, strings that cast to NaN, arrays, and objects that don't have a `valueOf()` function
-    will all result in a [CastError](/docs/api.html#mongooseerror_MongooseError.CastError).
+    will all result in a [CastError](/docs/validation.html#cast-errors) once validated, meaning that it will not throw on initialization, only when validated.
 
     <h4 id="dates">Dates</h4>
 
@@ -472,7 +472,7 @@ block content
     * `'0'`
     * `'no'`
 
-    Any other value causes a [CastError](/docs/api.html#mongooseerror_MongooseError.CastError).
+    Any other value causes a [CastError](/docs/validation.html#cast-errors).
     You can modify what values Mongoose converts to true or false using the
     `convertToTrue` and `convertToFalse` properties, which are [JavaScript sets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set).
 

index.pug

@@ -343,6 +343,9 @@ html(lang='en')
               <a href="https://casinohex.co.za/online-casinos/">
                 <img class="sponsor" src="https://images.opencollective.com/casinohex-co-za/470843d/logo/256.png" style="height: 100px">
               </a>
+              <a href="https://www.casinofever.ca/">
+                <img class="sponsor" src="https://images.opencollective.com/casinofever-ca1/4ad150e/logo/256.png" style="height: 100px">
+              </a>
             </div>
           </div>
 

lib/document.js

@@ -2606,7 +2606,8 @@ Document.prototype.$markValid = function(path) {
 };
 
 /**
- * Saves this document.
+ * Saves this document by inserting a new document into the database if [document.isNew](/docs/api.html#document_Document-isNew) is `true`,
+ * or sends an [updateOne](/docs/api.html#document_Document-updateOne) operation **only** with the modifications to the database, it does not replace the whole document in the latter case.
  *
  * ####Example:
  *

lib/helpers/populate/getModelsMapForPopulate.js

@@ -42,6 +42,14 @@ module.exports = function getModelsMapForPopulate(model, docs, options) {
     doc = docs[i];
 
     schema = getSchemaTypes(modelSchema, doc, options.path);
+    // Special case: populating a path that's a DocumentArray unless
+    // there's an explicit `ref` or `refPath` re: gh-8946
+    if (schema != null &&
+        schema.$isMongooseDocumentArray &&
+        schema.options.ref == null &&
+        schema.options.refPath == null) {
+      continue;
+    }
     const isUnderneathDocArray = schema && schema.$isUnderneathDocArray;
     if (isUnderneathDocArray && get(options, 'options.sort') != null) {
       return new MongooseError('Cannot populate with `sort` on path ' + options.path +

lib/helpers/update/applyTimestampsToChildren.js

@@ -165,6 +165,8 @@ function applyTimestampsToDocumentArray(arr, schematype, now) {
     if (createdAt != null) {
       arr[i][createdAt] = now;
     }
+
+    applyTimestampsToChildren(now, arr[i], schematype.schema);
   }
 }
 
@@ -182,4 +184,6 @@ function applyTimestampsToSingleNested(subdoc, schematype, now) {
   if (createdAt != null) {
     subdoc[createdAt] = now;
   }
+
+  applyTimestampsToChildren(now, subdoc, schematype.schema);
 }

lib/model.js

@@ -415,7 +415,8 @@ function generateVersionError(doc, modifiedPaths) {
 }
 
 /**
- * Saves this document.
+ * Saves this document by inserting a new document into the database if [document.isNew](/docs/api.html#document_Document-isNew) is `true`,
+ * or sends an [updateOne](/docs/api.html#document_Document-updateOne) operation **only** with the modifications to the database, it does not replace the whole document in the latter case.
  *
  * ####Example:
  *
@@ -3380,6 +3381,19 @@ Model.$__insertMany = function(arr, options, callback) {
 
     _this.collection.insertMany(docObjects, options, function(error, res) {
       if (error) {
+        // `writeErrors` is a property reported by the MongoDB driver,
+        // just not if there's only 1 error.
+        if (error.writeErrors == null &&
+            get(error, 'result.result.writeErrors') != null) {
+          error.writeErrors = error.result.result.writeErrors;
+        }
+
+        // `insertedDocs` is a Mongoose-specific property
+        const erroredIndexes = new Set(error.writeErrors.map(err => err.index));
+        error.insertedDocs = docAttributes.filter((doc, i) => {
+          return !erroredIndexes.has(i);
+        });
+
         callback(error, null);
         return;
       }
@@ -4444,6 +4458,14 @@ function populate(model, docs, options, callback) {
   }
 
   if (!hasOne) {
+    // If no models to populate but we have a nested populate,
+    // keep trying, re: gh-8946
+    if (options.populate != null) {
+      const opts = options.populate.map(pop => Object.assign({}, pop, {
+        path: options.path + '.' + pop.path
+      }));
+      return model.populate(docs, opts, callback);
+    }
     return callback();
   }
 

lib/schema.js

@@ -539,7 +539,6 @@ Schema.prototype.add = function add(obj, prefix) {
  * - _posts
  * - _pres
  * - collection
- * - db
  * - emit
  * - errors
  * - get
@@ -577,7 +576,6 @@ reserved.on =
 reserved.removeListener =
 // document properties and functions
 reserved.collection =
-reserved.db =
 reserved.errors =
 reserved.get =
 reserved.init =
@@ -659,21 +657,24 @@ Schema.prototype.path = function(path, obj) {
   const subpaths = path.split(/\./);
   const last = subpaths.pop();
   let branch = this.tree;
+  let fullPath = '';
 
-  subpaths.forEach(function(sub, i) {
+  for (const sub of subpaths) {
+    fullPath = fullPath += (fullPath.length > 0 ? '.' : '') + sub;
     if (!branch[sub]) {
+      this.nested[fullPath] = true;
       branch[sub] = {};
     }
     if (typeof branch[sub] !== 'object') {
       const msg = 'Cannot set nested path `' + path + '`. '
           + 'Parent path `'
-          + subpaths.slice(0, i).concat([sub]).join('.')
+          + fullPath
           + '` already set to type ' + branch[sub].name
           + '.';
       throw new Error(msg);
     }
     branch = branch[sub];
-  });
+  }
 
   branch[last] = utils.clone(obj);
 

lib/types/core_array.js

@@ -879,7 +879,7 @@ class CoreMongooseArray extends Array {
    *
    * ####Note:
    *
-   * _marks the entire array as modified, which if saved, will store it as a `$set` operation, potentially overwritting any changes that happen between when you retrieved the object and when you save it._
+   * _marks the entire array as modified, which if saved, will store it as a `$set` operation, potentially overwriting any changes that happen between when you retrieved the object and when you save it._
    *
    * @api public
    * @method unshift
@@ -889,8 +889,14 @@ class CoreMongooseArray extends Array {
   unshift() {
     _checkManualPopulation(this, arguments);
 
-    let values = [].map.call(arguments, this._cast, this);
-    values = this[arraySchemaSymbol].applySetters(values, this[arrayParentSymbol]);
+    let values;
+    if (this[arraySchemaSymbol] == null) {
+      values = arguments;
+    } else {
+      values = [].map.call(arguments, this._cast, this);
+      values = this[arraySchemaSymbol].applySetters(values, this[arrayParentSymbol]);
+    }
+
     [].unshift.apply(this, values);
     this._registerAtomic('$set', this);
     this._markModified();

test/model.populate.test.js

@@ -9378,4 +9378,34 @@ describe('model: populate:', function() {
       });
     });
   });
+
+  it('no-op if populating on a document array with no ref (gh-8946)', function() {
+    const teamSchema = Schema({
+      members: [{ user: { type: ObjectId, ref: 'User' } }]
+    });
+    const userSchema = Schema({ name: { type: String } });
+    userSchema.virtual('teams', {
+      ref: 'Team',
+      localField: '_id',
+      foreignField: 'members.user',
+      justOne: false
+    });
+    const User = db.model('User', userSchema);
+    const Team = db.model('Team', teamSchema);
+
+    return co(function*() {
+      const user = yield User.create({ name: 'User' });
+      yield Team.create({ members: [{ user: user._id }] });
+
+      const res = yield User.findOne().populate({
+        path: 'teams',
+        populate: {
+          path: 'members', // No ref
+          populate: { path: 'user' }
+        }
+      });
+
+      assert.equal(res.teams[0].members[0].user.name, 'User');
+    });
+  });
 });