SERVER-6860 Assertion: 13548:BufBuilder grow() > 64MB when trying to profile a huge document

Changed objsize() to size()

Author: renctan

src/mongo/db/client.cpp

@@ -693,17 +693,54 @@ namespace mongo {
 
 #define OPDEBUG_APPEND_NUMBER(x) if( x != -1 ) b.appendNumber( #x , (x) )
 #define OPDEBUG_APPEND_BOOL(x) if( x ) b.appendBool( #x , (x) )
-    void OpDebug::append( const CurOp& curop, BSONObjBuilder& b ) const {
+    bool OpDebug::append(const CurOp& curop, BSONObjBuilder& b, size_t maxSize) const {
         b.append( "op" , iscommand ? "command" : opToString( op ) );
         b.append( "ns" , ns.toString() );
-        if ( ! query.isEmpty() )
-            b.append( iscommand ? "command" : "query" , query );
-        else if ( ! iscommand && curop.haveQuery() )
-            curop.appendQuery( b , "query" );
-
-        if ( ! updateobj.isEmpty() )
-            b.append( "updateobj" , updateobj );
 
+        int queryUpdateObjSize = 0;
+        if (!query.isEmpty()) {
+            queryUpdateObjSize += query.objsize();
+        }
+        else if (!iscommand && curop.haveQuery()) {
+            queryUpdateObjSize += curop.query()["query"].size();
+        }
+
+        if (!updateobj.isEmpty()) {
+            queryUpdateObjSize += updateobj.objsize();
+        }
+
+        if (static_cast<size_t>(queryUpdateObjSize) > maxSize) {
+            if (!query.isEmpty()) {
+                // Use 60 since BSONObj::toString can truncate strings into 150 chars
+                // and we want to have enough room for both query and updateobj when
+                // the entire document is going to be serialized into a string
+                const string abbreviated(query.toString(false, false), 0, 60);
+                b.append(iscommand ? "command" : "query", abbreviated + "...");
+            }
+            else if (!iscommand && curop.haveQuery()) {
+                const string abbreviated(curop.query()["query"].toString(false, false), 0, 60);
+                b.append("query", abbreviated + "...");
+            }
+
+            if (!updateobj.isEmpty()) {
+                const string abbreviated(updateobj.toString(false, false), 0, 60);
+                b.append("updateobj", abbreviated + "...");
+            }
+
+            return false;
+        }
+
+        if (!query.isEmpty()) {
+            b.append(iscommand ? "command" : "query", query);
+        }
+        else if (!iscommand && curop.haveQuery()) {
+            curop.appendQuery(b, "query");
+        }
+
+        if (!updateobj.isEmpty()) {
+            b.append("updateobj", updateobj);
+        }
+
         const bool moved = (nmoved >= 1);
 
         OPDEBUG_APPEND_NUMBER( cursorid );
@@ -724,14 +761,15 @@ namespace mongo {
 
         b.appendNumber( "numYield" , curop.numYields() );
         b.append( "lockStats" , curop.lockStat().report() );
-        
-        if ( ! exceptionInfo.empty() ) 
+
+        if ( ! exceptionInfo.empty() )
             exceptionInfo.append( b , "exception" , "exceptionCode" );
-        
+
         OPDEBUG_APPEND_NUMBER( nreturned );
         OPDEBUG_APPEND_NUMBER( responseLength );
         b.append( "millis" , executionTime );
-        
+
+        return true;
     }
 
 }

src/mongo/db/curop.h

@@ -39,7 +39,21 @@ namespace mongo {
         void reset();
 
         string report( const CurOp& curop ) const;
-        void append( const CurOp& curop, BSONObjBuilder& b ) const;
+
+        /**
+         * Appends stored data and information from curop to the builder.
+         *
+         * @param curop information about the current operation which will be
+         *     use to append data to the builder.
+         * @param builder the BSON builder to use for appending data. Data can
+         *     still be appended even if this method returns false.
+         * @param maxSize the maximum allowed combined size for the query object
+         *     and update object
+         *
+         * @return false if the sum of the sizes for the query object and update
+         *     object exceeded maxSize
+         */
+        bool append(const CurOp& curop, BSONObjBuilder& builder, size_t maxSize) const;
 
         // -------------------
 
@@ -152,7 +166,7 @@ namespace mongo {
         ~CurOp();
 
         bool haveQuery() const { return _query.have(); }
-        BSONObj query() { return _query.get();  }
+        BSONObj query() const { return _query.get();  }
         void appendQuery( BSONObjBuilder& b , const StringData& name ) const { _query.append( b , name ); }
 
         void ensureStarted();

src/mongo/db/introspect.cpp

@@ -25,6 +25,10 @@
 #include "pdfile.h"
 #include "curop.h"
 
+namespace {
+    const size_t MAX_PROFILE_DOC_SIZE_BYTES = 100*1024;
+}
+
 namespace mongo {
 
     BufBuilder profileBufBuilder; // reused, instead of allocated every time - avoids a malloc/free cycle
@@ -39,20 +43,23 @@ namespace mongo {
         // build object
         profileBufBuilder.reset();
         BSONObjBuilder b(profileBufBuilder);
-        b.appendDate("ts", jsTime());
-        currentOp.debug().append( currentOp , b );
 
-        b.append("client", c.clientAddress() );
+        const bool isQueryObjTooBig = !currentOp.debug().append(currentOp, b,
+                MAX_PROFILE_DOC_SIZE_BYTES);
+
+        b.appendDate("ts", jsTime());
+        b.append("client", c.clientAddress());
 
-        if ( c.getAuthenticationInfo() )
-            b.append( "user" , c.getAuthenticationInfo()->getUser( nsToDatabase( ns ) ) );
+        if (c.getAuthenticationInfo()) {
+            b.append("user", c.getAuthenticationInfo()->getUser(nsToDatabase(ns)));
+        }
 
         BSONObj p = b.done();
 
-        if (p.objsize() > 100*1024){
+        if (static_cast<size_t>(p.objsize()) > MAX_PROFILE_DOC_SIZE_BYTES || isQueryObjTooBig) {
             string small = p.toString(/*isArray*/false, /*full*/false);
 
-            warning() << "can't add full line to system.profile: " << small;
+            warning() << "can't add full line to system.profile: " << small << endl;
 
             // rebuild with limited info
             BSONObjBuilder b(profileBufBuilder);
@@ -62,7 +69,9 @@ namespace mongo {
                 b.append( "user" , c.getAuthenticationInfo()->getUser( nsToDatabase( ns ) ) );
 
             b.append("err", "profile line too large (max is 100KB)");
-            if (small.size() < 100*1024){ // should be much smaller but if not don't break anything
+
+            // should be much smaller but if not don't break anything
+            if (small.size() < MAX_PROFILE_DOC_SIZE_BYTES){
                 b.append("abbreviated", small);
             }
 

src/mongo/dbtests/profile_test.cpp

@@ -0,0 +1,109 @@
+/**
+ *    Copyright (C) 2012 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * This file contains tests for the profile command
+ */
+
+#include "mongo/db/instance.h"
+#include "mongo/unittest/unittest.h"
+
+using mongo::BSONObj;
+using mongo::DBDirectClient;
+
+using std::string;
+
+namespace mongo_test {
+    class Profiler: public mongo::unittest::Test {
+    public:
+        static const string PROFILER_TEST_DB;
+        static const string PROFILER_TEST_NS;
+        static const string PROFILE_NS;
+
+    protected:
+        void setUp() {
+            BSONObj ret;
+            db.runCommand(PROFILER_TEST_DB, BSON("dropDatabase" << 1), ret);
+            ASSERT(ret["ok"].trueValue());
+        }
+
+        DBDirectClient db;
+    };
+
+    const string Profiler::PROFILER_TEST_DB = "profilerTestDB";
+    const string Profiler::PROFILER_TEST_NS = Profiler::PROFILER_TEST_DB + ".test";
+    const string Profiler::PROFILE_NS = Profiler::PROFILER_TEST_DB + ".system.profile";
+
+    TEST_F(Profiler, BigDoc) {
+        // Test that update with large document with a long string can be
+        // be profiled in a shortened version
+        const string bigStr(16 * (1 << 20) - 200, 'a');
+
+        {
+            BSONObj replyObj;
+            db.runCommand(PROFILER_TEST_DB, BSON("profile" << 2), replyObj);
+            ASSERT(replyObj["ok"].trueValue());
+        }
+
+        const BSONObj doc(BSON("field" << bigStr));
+        db.update(PROFILER_TEST_NS, doc, doc);
+
+        std::auto_ptr<mongo::DBClientCursor> cursor = db.query(PROFILE_NS, BSONObj());
+        ASSERT(cursor->more());
+
+        BSONObj profileDoc(cursor->next());
+        ASSERT(profileDoc.hasField("abbreviated"));
+        const string abbreviatedField(profileDoc["abbreviated"].str());
+
+        // Make sure that the abbreviated field contains the query and the updateobj info
+        ASSERT(abbreviatedField.find("query:") != string::npos);
+        ASSERT(abbreviatedField.find("updateobj:") != string::npos);
+    }
+
+    TEST_F(Profiler, BigDocWithManyFields) {
+        // Test that update with large document with lots of fields can be
+        // be profiled in a shortened version
+        mongo::BSONObjBuilder builder;
+
+        for (int x = 0; x < (1 << 20); x++) {
+            const string fieldName(mongo::str::stream() << "x" << x);
+            builder.append(fieldName, x);
+        }
+
+        DBDirectClient db;
+
+        {
+            BSONObj replyObj;
+            db.runCommand(PROFILER_TEST_DB, BSON("profile" << 2), replyObj);
+            ASSERT(replyObj["ok"].trueValue());
+        }
+
+        const BSONObj doc(builder.done());
+        db.update(PROFILER_TEST_NS, doc, doc);
+
+        std::auto_ptr<mongo::DBClientCursor> cursor = db.query(PROFILE_NS, BSONObj());
+        ASSERT(cursor->more());
+
+        BSONObj profileDoc(cursor->next());
+        ASSERT(profileDoc.hasField("abbreviated"));
+        const string abbreviatedField(profileDoc["abbreviated"].str());
+
+        // Make sure that the abbreviated field contains the query and the updateobj info
+        ASSERT(abbreviatedField.find("query:") != string::npos);
+        ASSERT(abbreviatedField.find("updateobj:") != string::npos);
+    }
+}
+