BUG: fix bugs in tests, more resampling

Author: wesm

pandas/core/groupby.py

@@ -796,7 +796,7 @@ def get_grouper(self, obj):
 class BinGrouper(Grouper):
 
     def __init__(self, bins, binlabels, filter_empty=False):
-        self.bins = bins
+        self.bins = com._ensure_int32(bins)
         self.binlabels = _ensure_index(binlabels)
         self._filter_empty_groups = filter_empty
 
@@ -1048,6 +1048,8 @@ def _get_grouper(obj, key=None, axis=0, level=None, sort=True):
     if isinstance(key, CustomGrouper):
         gpr = key.get_grouper(obj)
         return gpr, []
+    elif isinstance(key, Grouper):
+        return key, []
 
     if not isinstance(key, (tuple, list)):
         keys = [key]

pandas/core/internals.py

@@ -232,6 +232,12 @@ def interpolate(self, method='pad', axis=0, inplace=False, limit=None):
 
         return make_block(values, self.items, self.ref_items)
 
+    def take(self, indexer, axis=1, fill_value=np.nan):
+        assert(axis >= 1)
+        new_values = com.take_fast(self.values, indexer, None,
+                                   None, axis=axis,
+                                   fill_value=fill_value)
+        return make_block(new_values, self.items, self.ref_items)
 
 #-------------------------------------------------------------------------------
 # Is this even possible?

pandas/src/groupby.pyx

@@ -607,6 +607,9 @@ def generate_bins_dt64(ndarray[int64_t] values, ndarray[int64_t] binner,
         bins[bc] = j
         bc += 1
 
+    # if len(bins) > 0 and bins[-1] == lenidx:
+    #     bins = bins[:-1]
+
     return bins
 
 # add passing bin edges, instead of labels
@@ -628,7 +631,10 @@ def group_add_bin(ndarray[float64_t, ndim=2] out,
     nobs = np.zeros_like(out)
     sumx = np.zeros_like(out)
 
-    ngroups = len(bins) + 1
+    if bins[len(bins) - 1] == len(values):
+        ngroups = len(bins)
+    else:
+        ngroups = len(bins) + 1
     N, K = (<object> values).shape
 
     b = 0
@@ -660,7 +666,7 @@ def group_add_bin(ndarray[float64_t, ndim=2] out,
 
     for i in range(ngroups):
         for j in range(K):
-            if nobs[i] == 0:
+            if nobs[i, j] == 0:
                 out[i, j] = nan
             else:
                 out[i, j] = sumx[i, j]
@@ -682,7 +688,10 @@ def group_prod_bin(ndarray[float64_t, ndim=2] out,
     nobs = np.zeros_like(out)
     prodx = np.ones_like(out)
 
-    ngroups = len(bins) + 1
+    if bins[len(bins) - 1] == len(values):
+        ngroups = len(bins)
+    else:
+        ngroups = len(bins) + 1
     N, K = (<object> values).shape
 
     b = 0
@@ -714,7 +723,7 @@ def group_prod_bin(ndarray[float64_t, ndim=2] out,
 
     for i in range(ngroups):
         for j in range(K):
-            if nobs[i] == 0:
+            if nobs[i, j] == 0:
                 out[i, j] = nan
             else:
                 out[i, j] = prodx[i, j]
@@ -738,8 +747,11 @@ def group_min_bin(ndarray[float64_t, ndim=2] out,
     minx = np.empty_like(out)
     minx.fill(np.inf)
 
+    if bins[len(bins) - 1] == len(values):
+        ngroups = len(bins)
+    else:
+        ngroups = len(bins) + 1
 
-    ngroups = len(bins) + 1
     N, K = (<object> values).shape
 
     b = 0
@@ -773,7 +785,7 @@ def group_min_bin(ndarray[float64_t, ndim=2] out,
 
     for i in range(ngroups):
         for j in range(K):
-            if nobs[i] == 0:
+            if nobs[i, j] == 0:
                 out[i, j] = nan
             else:
                 out[i, j] = minx[i, j]
@@ -796,7 +808,11 @@ def group_max_bin(ndarray[float64_t, ndim=2] out,
     maxx = np.empty_like(out)
     maxx.fill(-np.inf)
 
-    ngroups = len(bins) + 1
+    if bins[len(bins) - 1] == len(values):
+        ngroups = len(bins)
+    else:
+        ngroups = len(bins) + 1
+
     N, K = (<object> values).shape
 
     b = 0
@@ -830,7 +846,7 @@ def group_max_bin(ndarray[float64_t, ndim=2] out,
 
     for i in range(ngroups):
         for j in range(K):
-            if nobs[i] == 0:
+            if nobs[i, j] == 0:
                 out[i, j] = nan
             else:
                 out[i, j] = maxx[i, j]
@@ -851,7 +867,11 @@ def group_ohlc(ndarray[float64_t, ndim=2] out,
         float64_t vopen, vhigh, vlow, vclose, NA
         bint got_first = 0
 
-    ngroups = len(bins) + 1
+    if bins[len(bins) - 1] == len(values):
+        ngroups = len(bins)
+    else:
+        ngroups = len(bins) + 1
+
     N, K = (<object> values).shape
 
     if out.shape[1] != 4:
@@ -922,7 +942,10 @@ def group_mean_bin(ndarray[float64_t, ndim=2] out,
     sumx = np.zeros_like(out)
 
     N, K = (<object> values).shape
-    ngroups = len(bins) + 1
+    if bins[len(bins) - 1] == len(values):
+        ngroups = len(bins)
+    else:
+        ngroups = len(bins) + 1
 
     b = 0
     if K > 1:
@@ -953,8 +976,8 @@ def group_mean_bin(ndarray[float64_t, ndim=2] out,
 
     for i in range(ngroups):
         for j in range(K):
-            count = nobs[i]
-            if nobs[i] == 0:
+            count = nobs[i, j]
+            if nobs[i, j] == 0:
                 out[i, j] = nan
             else:
                 out[i, j] = sumx[i, j] / count
@@ -975,7 +998,11 @@ def group_var_bin(ndarray[float64_t, ndim=2] out,
     sumx = np.zeros_like(out)
     sumxx = np.zeros_like(out)
 
-    ngroups = len(bins) + 1
+    if bins[len(bins) - 1] == len(values):
+        ngroups = len(bins)
+    else:
+        ngroups = len(bins) + 1
+
     N, K = (<object> values).shape
 
     b = 0
@@ -1010,7 +1037,7 @@ def group_var_bin(ndarray[float64_t, ndim=2] out,
 
     for i in range(ngroups):
         for j in range(K):
-            ct = nobs[i]
+            ct = nobs[i, j]
             if ct < 2:
                 out[i, j] = nan
             else:

pandas/tests/test_tseries.py

@@ -289,14 +289,15 @@ def test_generate_bins():
         bins = func(values, binner, closed='left')
         assert((bins == np.array([2, 5, 6])).all())
 
-        bins = func(values, binner, closed='left')
-        assert((bins == np.array([2, 5, 6])).all())
-
         bins = func(values, binner, closed='right')
         assert((bins == np.array([3, 6, 6])).all())
 
+    for func in [lib.generate_bins_dt64, generate_bins_generic]:
+        values = np.array([1,2,3,4,5,6], dtype=np.int64)
+        binner = np.array([0,3,6], dtype=np.int64)
+
         bins = func(values, binner, closed='right')
-        assert((bins == np.array([3, 6, 6])).all())
+        assert((bins == np.array([3, 6])).all())
 
 class TestBinGroupers(unittest.TestCase):
 
@@ -338,22 +339,22 @@ def _check_versions(self, irr_func, bin_func, np_func):
 
         assert_almost_equal(out, exp)
 
-        # duplicate bins
         bins = np.array([3, 9, 10], dtype=np.int32)
-        out  = np.zeros((4, 1), np.float64)
+        out  = np.zeros((3, 1), np.float64)
         counts = np.zeros(len(out), dtype=np.int32)
         bin_func(out, counts, obj, bins)
         exp = np.array([np_func(obj[:3]), np_func(obj[3:9]),
-                        np_func(obj[9:]), np.nan],
+                        np_func(obj[9:])],
                        dtype=np.float64)
         assert_almost_equal(out.squeeze(), exp)
 
+        # duplicate bins
         bins = np.array([3, 6, 10, 10], dtype=np.int32)
-        out  = np.zeros((5, 1), np.float64)
+        out  = np.zeros((4, 1), np.float64)
         counts = np.zeros(len(out), dtype=np.int32)
         bin_func(out, counts, obj, bins)
         exp = np.array([np_func(obj[:3]), np_func(obj[3:6]),
-                        np_func(obj[6:10]), np.nan, np.nan],
+                        np_func(obj[6:10]), np.nan],
                        dtype=np.float64)
         assert_almost_equal(out.squeeze(), exp)
 

pandas/tseries/resample.py

@@ -73,34 +73,33 @@ def _get_time_grouper(self, obj):
 
         if self.kind is None or self.kind == 'timestamp':
             binner, bins, binlabels = self._get_time_bins(axis)
-            grouper = BinGrouper(bins, binlabels)
         else:
-            index = binner = PeriodIndex(start=axis[0], end=axis[-1],
-                                         freq=self.freq)
-
-            end_stamps = (index + 1).asfreq('D', 's').to_timestamp()
-            bins = axis.searchsorted(end_stamps, side='left')
-
-            grouper = BinGrouper(bins, index)
+            binner, bins, binlabels = self._get_time_period_bins(axis)
 
+        grouper = BinGrouper(bins, binlabels)
         return binner, grouper
 
     def _get_time_bins(self, axis):
         return _make_time_bins(axis, self.freq, begin=self.begin,
-                               end=self.end, nperiods=self.nperiods,
-                               closed=self.closed, label=self.label)
+                               end=self.end, closed=self.closed,
+                               label=self.label)
+
+    def _get_time_period_bins(self, axis):
+        return _make_period_bins(axis, self.freq, begin=self.begin,
+                                 end=self.end, closed=self.closed,
+                                 label=self.label)
 
     def _resample_timestamps(self, obj):
-        axis = obj._get_axis(self.axis)
+        axlabels = obj._get_axis(self.axis)
 
         binner, grouper = self._get_time_grouper(obj)
 
         # downsamples
-        if len(grouper.binlabels) < len(axis):
-            grouped  = obj.groupby(grouper, axis=axis)
+        if len(grouper.binlabels) < len(axlabels):
+            grouped  = obj.groupby(grouper, axis=self.axis)
             result = grouped.agg(self.how)
         else:
-            assert(axis == 0)
+            assert(self.axis == 0)
             # upsampling
 
             # this is sort of a hack
@@ -115,12 +114,12 @@ def _resample_timestamps(self, obj):
         return result
 
     def _resample_periods(self, obj):
-        axis = obj._get_axis(self.axis)
+        axlabels = obj._get_axis(self.axis)
 
         # Start vs. end of period
-        memb = axis.asfreq(self.freq, how=self.convention)
+        memb = axlabels.asfreq(self.freq, how=self.convention)
 
-        if is_subperiod(self.axis.freq, self.freq):
+        if is_subperiod(axlabels.freq, self.freq):
             # Downsampling
             if len(memb) > 1:
                 rng = np.arange(memb.values[0], memb.values[-1])
@@ -131,9 +130,9 @@ def _resample_periods(self, obj):
             index = period_range(memb[0], memb[-1], freq=self.freq)
             grouper = BinGrouper(bins, index)
 
-            grouped = obj.groupby(grouper, axis=axis)
+            grouped = obj.groupby(grouper, axis=self.axis)
             return grouped.agg(self.how)
-        elif is_superperiod(self.axis.freq, self.freq):
+        elif is_superperiod(axlabels.freq, self.freq):
             # Generate full range
             new_index = period_range(memb[0], memb[-1], freq=self.freq)
 
@@ -144,7 +143,7 @@ def _resample_periods(self, obj):
             return _take_new_index(obj, indexer, new_index, axis=self.axis)
         else:
             raise ValueError('Frequency %s cannot be resampled to %s'
-                             % (self.axis.freq, self.freq))
+                             % (axlabels.freq, self.freq))
 
 
 def _take_new_index(obj, indexer, new_index, axis=0):
@@ -168,31 +167,47 @@ def _take_new_index(obj, indexer, new_index, axis=0):
         raise NotImplementedError
 
 
-def _make_period_bins(axis, freq):
-    index = PeriodIndex(start=axis[0], end=axis[-1], freq=freq)
-    end_stamps = (index + 1).asfreq('D', 's').to_timestamp()
+def _make_period_bins(axis, freq, begin=None, end=None,
+                    closed='right', label='right'):
+    assert(isinstance(axis, DatetimeIndex))
+
+    if len(axis) == 0:
+        # TODO: Should we be a bit more careful here?
+        return [], [], []
+
+    first, last = _get_range_edges(axis, begin, end, freq, closed=closed)
+    binlabels = binner = PeriodIndex(start=first, end=last, freq=freq)
+
+    # a little hack
+    trimmed = False
+    if len(binner) > 2 and binner[-2] == axis[-1]:
+        binner = binner[:-1]
+        trimmed = True
+
+    end_stamps = (binlabels + 1).asfreq('D', 's').to_timestamp()
     bins = axis.searchsorted(end_stamps, side='left')
 
-    return index, bins, index
+    if label == 'right':
+        bins = bins[1:]
+        labels = binner[1:]
+    elif not trimmed:
+        labels = binner[:-1]
+    else:
+        labels = binner
+
+    return binner, bins, labels
 
 
-def _make_time_bins(axis, freq, begin=None, end=None, nperiods=None,
+def _make_time_bins(axis, freq, begin=None, end=None,
                     closed='right', label='right'):
     assert(isinstance(axis, DatetimeIndex))
 
     if len(axis) == 0:
         # TODO: Should we be a bit more careful here?
         return [], [], []
 
-    if isinstance(freq, basestring):
-        freq = to_offset(freq)
-
-    if not isinstance(freq, DateOffset):
-        raise ValueError("Rule not a recognized offset")
-
     first, last = _get_range_edges(axis, begin, end, freq, closed=closed)
-    binner = DatetimeIndex(freq=freq, start=first, end=last,
-                           periods=nperiods)
+    binner = DatetimeIndex(freq=freq, start=first, end=last)
 
     # a little hack
     trimmed = False
@@ -213,6 +228,12 @@ def _make_time_bins(axis, freq, begin=None, end=None, nperiods=None,
     return binner, bins, labels
 
 def _get_range_edges(axis, begin, end, offset, closed='left'):
+    if isinstance(offset, basestring):
+        offset = to_offset(offset)
+
+    if not isinstance(offset, DateOffset):
+        raise ValueError("Rule not a recognized offset")
+
     if begin is None:
         if closed == 'left':
             first = Timestamp(offset.rollback(axis[0]))