diff --git a/pandas/tools/pivot.py b/pandas/tools/pivot.py
index d699639c6c796..47ee9bc521f92 100644
--- a/pandas/tools/pivot.py
+++ b/pandas/tools/pivot.py
@@ -267,7 +267,7 @@ def _all_key(key):
             return (key, margins_name) + ('',) * (len(cols) - 1)
 
         if len(rows) > 0:
-            margin = data[rows + values].groupby(rows).agg(aggfunc)
+            margin = data[data[cols].notnull().any(axis=1)][rows + values].groupby(rows).agg(aggfunc)
             cat_axis = 1
 
             for key, piece in table.groupby(level=0, axis=cat_axis):
@@ -304,7 +304,7 @@ def _all_key(key):
         margin_keys = table.columns
 
     if len(cols) > 0:
-        row_margin = data[cols + values].groupby(cols).agg(aggfunc)
+        row_margin = data[data[rows].notnull().any(axis=1)][cols + values].groupby(cols).agg(aggfunc)
         row_margin = row_margin.stack()
 
         # slight hack
@@ -462,4 +462,4 @@ def _get_names(arrs, names, prefix='row'):
         if not isinstance(names, list):
             names = list(names)
 
-    return names
+    return names
\ No newline at end of file