+-https://groups.google.com/forum/#!forum/pydata +[Go to Top](#table-of-contents) diff --git a/RELEASE.md b/RELEASE.md deleted file mode 100644 index 23c1817b7647c..0000000000000 --- a/RELEASE.md +++ /dev/null @@ -1,6 +0,0 @@ -Release Notes -============= - -The list of changes to pandas between each release can be found -[here](http://pandas.pydata.org/pandas-docs/stable/whatsnew.html). For full -details, see the commit logs at http://github.com/pydata/pandas. diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 9cec7895f1493..0000000000000 --- a/appveyor.yml +++ /dev/null @@ -1,38 +0,0 @@ -environment: - global: - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script intepreter - # See: http://stackoverflow.com/a/13751649/163740 - CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci\\run_with_env.cmd" - - matrix: - - PYTHON: "C:\\Python27_32" - PYTHON_VERSION: "2.7" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python27_64" - PYTHON_VERSION: "2.7" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python34_32" - PYTHON_VERSION: "3.4" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python34_64" - PYTHON_VERSION: "3.4" - PYTHON_ARCH: "64" - -install: - # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit), - # as well as pip, conda-build, and the binstar CLI - - echo "install" - - cd - - ls -ltr - - powershell .\\ci\\install_appveyor.ps1 - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - -build: false - -test_script: - - "%CMD_IN_ENV% %PYTHON%/python.exe setup.py build_ext --inplace" - - "%PYTHON%/Scripts/nosetests -A \"not slow and not network and not disabled\" pandas" diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json new file mode 100644 index 0000000000000..1a288e399a699 --- /dev/null +++ b/asv_bench/asv.conf.json @@ -0,0 +1,132 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "pandas", + + // The project's homepage + "project_url": "https://pandas.pydata.org/", + + // The URL of the source code repository for the project being + // benchmarked + "repo": "..", + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "default" (for mercurial). + "branches": ["main"], + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "conda", + + // the base URL to show a commit for the project. + "show_commit_url": "https://github.com/pandas-dev/pandas/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["3.11"], + + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list or empty string indicates to just test against the default + // (latest) version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed via + // pip (with all the conda available packages installed first, + // followed by the pip installed packages). + "matrix": { + "pip+build": [], + "Cython": [], + "matplotlib": [], + "sqlalchemy": [], + "scipy": [], + "numba": [], + "numexpr": [], + "pytables": [null, ""], // platform dependent, see excludes below + "pyarrow": [], + "tables": [null, ""], + "openpyxl": [], + "xlsxwriter": [], + "xlrd": [], + "odfpy": [], + "jinja2": [], + "meson": [], + "meson-python": [], + "python-build": [], + }, + "conda_channels": ["conda-forge"], + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + "exclude": [ + // On conda install pytables, otherwise tables + {"environment_type": "conda", "tables": ""}, + {"environment_type": "conda", "pytables": null}, + {"environment_type": "(?!conda).*", "tables": null}, + {"environment_type": "(?!conda).*", "pytables": ""}, + ], + "include": [], + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + // "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + // "env_dir": "env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + // "results_dir": "results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + // "html_dir": "html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache wheels of the recent builds in each + // environment, making them faster to install next time. This is + // number of builds to keep, per environment. + "build_cache_size": 8, + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + "regressions_first_commits": { + ".*": "0409521665" + }, + "regression_thresholds": { + }, + "build_command": + ["python -m build -Cbuilddir=builddir --wheel --outdir {build_cache_dir} {build_dir}"] +} diff --git a/asv_bench/benchmarks/__init__.py b/asv_bench/benchmarks/__init__.py new file mode 100644 index 0000000000000..eada147852fe1 --- /dev/null +++ b/asv_bench/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Pandas benchmarks.""" diff --git a/asv_bench/benchmarks/algorithms.py b/asv_bench/benchmarks/algorithms.py new file mode 100644 index 0000000000000..933e8fbc175d8 --- /dev/null +++ b/asv_bench/benchmarks/algorithms.py @@ -0,0 +1,210 @@ +from importlib import import_module + +import numpy as np + +import pandas as pd + +for imp in ["pandas.util", "pandas.tools.hashing"]: + try: + hashing = import_module(imp) + break + except (ImportError, TypeError, ValueError): + pass + + +class Factorize: + params = [ + [True, False], + [True, False], + [ + "int64", + "uint64", + "float64", + "object", + "object_str", + "datetime64[ns]", + "datetime64[ns, tz]", + "Int64", + "boolean", + "string[pyarrow]", + ], + ] + param_names = ["unique", "sort", "dtype"] + + def setup(self, unique, sort, dtype): + N = 10**5 + + if dtype in ["int64", "uint64", "Int64", "object"]: + data = pd.Index(np.arange(N), dtype=dtype) + elif dtype == "float64": + data = pd.Index(np.random.randn(N), dtype=dtype) + elif dtype == "boolean": + data = pd.array(np.random.randint(0, 2, N), dtype=dtype) + elif dtype == "datetime64[ns]": + data = pd.date_range("2011-01-01", freq="h", periods=N) + elif dtype == "datetime64[ns, tz]": + data = pd.date_range("2011-01-01", freq="h", periods=N, tz="Asia/Tokyo") + elif dtype == "object_str": + data = pd.Index([f"i-{i}" for i in range(N)], dtype=object) + elif dtype == "string[pyarrow]": + data = pd.array( + pd.Index([f"i-{i}" for i in range(N)], dtype=object), + dtype="string[pyarrow]", + ) + else: + raise NotImplementedError + + if not unique: + data = data.repeat(5) + self.data = data + + def time_factorize(self, unique, sort, dtype): + pd.factorize(self.data, sort=sort) + + def peakmem_factorize(self, unique, sort, dtype): + pd.factorize(self.data, sort=sort) + + +class Duplicated: + params = [ + [True, False], + ["first", "last", False], + [ + "int64", + "uint64", + "float64", + "string", + "datetime64[ns]", + "datetime64[ns, tz]", + "timestamp[ms][pyarrow]", + "duration[s][pyarrow]", + ], + ] + param_names = ["unique", "keep", "dtype"] + + def setup(self, unique, keep, dtype): + N = 10**5 + if dtype in ["int64", "uint64"]: + data = pd.Index(np.arange(N), dtype=dtype) + elif dtype == "float64": + data = pd.Index(np.random.randn(N), dtype="float64") + elif dtype == "string": + data = pd.Index([f"i-{i}" for i in range(N)], dtype=object) + elif dtype == "datetime64[ns]": + data = pd.date_range("2011-01-01", freq="h", periods=N) + elif dtype == "datetime64[ns, tz]": + data = pd.date_range("2011-01-01", freq="h", periods=N, tz="Asia/Tokyo") + elif dtype in ["timestamp[ms][pyarrow]", "duration[s][pyarrow]"]: + data = pd.Index(np.arange(N), dtype=dtype) + else: + raise NotImplementedError + if not unique: + data = data.repeat(5) + self.idx = data + # cache is_unique + self.idx.is_unique + + def time_duplicated(self, unique, keep, dtype): + self.idx.duplicated(keep=keep) + + +class DuplicatedMaskedArray: + params = [ + [True, False], + ["first", "last", False], + ["Int64", "Float64"], + ] + param_names = ["unique", "keep", "dtype"] + + def setup(self, unique, keep, dtype): + N = 10**5 + data = pd.Series(np.arange(N), dtype=dtype) + data[list(range(1, N, 100))] = pd.NA + if not unique: + data = data.repeat(5) + self.ser = data + # cache is_unique + self.ser.is_unique + + def time_duplicated(self, unique, keep, dtype): + self.ser.duplicated(keep=keep) + + +class Hashing: + def setup_cache(self): + N = 10**5 + + df = pd.DataFrame( + { + "strings": pd.Series( + pd.Index([f"i-{i}" for i in range(10000)], dtype=object).take( + np.random.randint(0, 10000, size=N) + ) + ), + "floats": np.random.randn(N), + "ints": np.arange(N), + "dates": pd.date_range("20110101", freq="s", periods=N), + "timedeltas": pd.timedelta_range("1 day", freq="s", periods=N), + } + ) + df["categories"] = df["strings"].astype("category") + df.iloc[10:20] = np.nan + return df + + def time_frame(self, df): + hashing.hash_pandas_object(df) + + def time_series_int(self, df): + hashing.hash_pandas_object(df["ints"]) + + def time_series_string(self, df): + hashing.hash_pandas_object(df["strings"]) + + def time_series_float(self, df): + hashing.hash_pandas_object(df["floats"]) + + def time_series_categorical(self, df): + hashing.hash_pandas_object(df["categories"]) + + def time_series_timedeltas(self, df): + hashing.hash_pandas_object(df["timedeltas"]) + + def time_series_dates(self, df): + hashing.hash_pandas_object(df["dates"]) + + +class Quantile: + params = [ + [0, 0.5, 1], + ["linear", "nearest", "lower", "higher", "midpoint"], + ["float64", "int64", "uint64"], + ] + param_names = ["quantile", "interpolation", "dtype"] + + def setup(self, quantile, interpolation, dtype): + N = 10**5 + if dtype in ["int64", "uint64"]: + data = np.arange(N, dtype=dtype) + elif dtype == "float64": + data = np.random.randn(N) + else: + raise NotImplementedError + self.ser = pd.Series(data.repeat(5)) + + def time_quantile(self, quantile, interpolation, dtype): + self.ser.quantile(quantile, interpolation=interpolation) + + +class SortIntegerArray: + params = [10**3, 10**5] + + def setup(self, N): + data = np.arange(N, dtype=float) + data[40] = np.nan + self.array = pd.array(data, dtype="Int64") + + def time_argsort(self, N): + self.array.argsort() + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/algos/__init__.py b/asv_bench/benchmarks/algos/__init__.py new file mode 100644 index 0000000000000..97c9ab09b9c6b --- /dev/null +++ b/asv_bench/benchmarks/algos/__init__.py @@ -0,0 +1,12 @@ +""" +algos/ directory is intended for individual functions from core.algorithms + +In many cases these algorithms are reachable in multiple ways: + algos.foo(x, y) + Series(x).foo(y) + Index(x).foo(y) + pd.array(x).foo(y) + +In most cases we profile the Series variant directly, trusting the performance +of the others to be highly correlated. +""" diff --git a/asv_bench/benchmarks/algos/isin.py b/asv_bench/benchmarks/algos/isin.py new file mode 100644 index 0000000000000..a17732c70c2c7 --- /dev/null +++ b/asv_bench/benchmarks/algos/isin.py @@ -0,0 +1,341 @@ +import numpy as np + +from pandas import ( + Categorical, + Index, + NaT, + Series, + date_range, +) + + +class IsIn: + params = [ + "int64", + "uint64", + "object", + "Int64", + "boolean", + "bool", + "datetime64[ns]", + "category[object]", + "category[int]", + "str", + "string[python]", + "string[pyarrow]", + ] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10000 + + self.mismatched = [NaT.to_datetime64()] * 2 + + if dtype in ["boolean", "bool"]: + self.series = Series(np.random.randint(0, 2, N)).astype(dtype) + self.values = [True, False] + + elif dtype == "datetime64[ns]": + # Note: values here is much larger than non-dt64ns cases + + # dti has length=115777 + dti = date_range(start="2015-10-26", end="2016-01-01", freq="50s") + self.series = Series(dti) + self.values = self.series._values[::3] + self.mismatched = [1, 2] + + elif dtype in ["category[object]", "category[int]"]: + # Note: sizes are different in this case than others + n = 5 * 10**5 + sample_size = 100 + + arr = list(np.random.randint(0, n // 10, size=n)) + if dtype == "category[object]": + arr = [f"s{i:04d}" for i in arr] + + self.values = np.random.choice(arr, sample_size) + self.series = Series(arr).astype("category") + + elif dtype in ["str", "string[python]", "string[pyarrow]"]: + try: + self.series = Series( + Index([f"i-{i}" for i in range(N)], dtype=object)._values, + dtype=dtype, + ) + except ImportError as err: + raise NotImplementedError from err + self.values = list(self.series[:2]) + + else: + self.series = Series(np.random.randint(1, 10, N)).astype(dtype) + self.values = [1, 2] + + self.cat_values = Categorical(self.values) + + def time_isin(self, dtype): + self.series.isin(self.values) + + def time_isin_categorical(self, dtype): + self.series.isin(self.cat_values) + + def time_isin_empty(self, dtype): + self.series.isin([]) + + def time_isin_mismatched_dtype(self, dtype): + self.series.isin(self.mismatched) + + +class IsinAlmostFullWithRandomInt: + params = [ + [np.float64, np.int64, np.uint64, np.object_], + range(10, 21), + ["inside", "outside"], + ] + param_names = ["dtype", "exponent", "title"] + + def setup(self, dtype, exponent, title): + M = 3 * 2 ** (exponent - 2) + # 0.77-the maximal share of occupied buckets + self.series = Series(np.random.randint(0, M, M)).astype(dtype) + + values = np.random.randint(0, M, M).astype(dtype) + if title == "inside": + self.values = values + elif title == "outside": + self.values = values + M + else: + raise ValueError(title) + + def time_isin(self, dtype, exponent, title): + self.series.isin(self.values) + + +class IsinWithRandomFloat: + params = [ + [np.float64, np.object_], + [ + 1_300, + 2_000, + 7_000, + 8_000, + 70_000, + 80_000, + 750_000, + 900_000, + ], + ["inside", "outside"], + ] + param_names = ["dtype", "size", "title"] + + def setup(self, dtype, size, title): + self.values = np.random.rand(size) + self.series = Series(self.values).astype(dtype) + np.random.shuffle(self.values) + + if title == "outside": + self.values = self.values + 0.1 + + def time_isin(self, dtype, size, title): + self.series.isin(self.values) + + +class IsinWithArangeSorted: + params = [ + [np.float64, np.int64, np.uint64, np.object_], + [ + 1_000, + 2_000, + 8_000, + 100_000, + 1_000_000, + ], + ] + param_names = ["dtype", "size"] + + def setup(self, dtype, size): + self.series = Series(np.arange(size)).astype(dtype) + self.values = np.arange(size).astype(dtype) + + def time_isin(self, dtype, size): + self.series.isin(self.values) + + +class IsinWithArange: + params = [ + [np.float64, np.int64, np.uint64, np.object_], + [ + 1_000, + 2_000, + 8_000, + ], + [-2, 0, 2], + ] + param_names = ["dtype", "M", "offset_factor"] + + def setup(self, dtype, M, offset_factor): + offset = int(M * offset_factor) + tmp = Series(np.random.randint(offset, M + offset, 10**6)) + self.series = tmp.astype(dtype) + self.values = np.arange(M).astype(dtype) + + def time_isin(self, dtype, M, offset_factor): + self.series.isin(self.values) + + +class IsInFloat64: + params = [ + [np.float64, "Float64"], + ["many_different_values", "few_different_values", "only_nans_values"], + ] + param_names = ["dtype", "title"] + + def setup(self, dtype, title): + N_many = 10**5 + N_few = 10**6 + self.series = Series([1, 2], dtype=dtype) + + if title == "many_different_values": + # runtime is dominated by creation of the lookup-table + self.values = np.arange(N_many, dtype=np.float64) + elif title == "few_different_values": + # runtime is dominated by creation of the lookup-table + self.values = np.zeros(N_few, dtype=np.float64) + elif title == "only_nans_values": + # runtime is dominated by creation of the lookup-table + self.values = np.full(N_few, np.nan, dtype=np.float64) + else: + raise ValueError(title) + + def time_isin(self, dtype, title): + self.series.isin(self.values) + + +class IsInForObjects: + """ + A subset of the cartesian product of cases have special motivations: + + "nans" x "nans" + if nan-objects are different objects, + this has the potential to trigger O(n^2) running time + + "short" x "long" + running time dominated by the preprocessing + + "long" x "short" + running time dominated by look-up + + "long" x "long" + no dominating part + + "long_floats" x "long_floats" + because of nans floats are special + no dominating part + + """ + + variants = ["nans", "short", "long", "long_floats"] + + params = [variants, variants] + param_names = ["series_type", "vals_type"] + + def setup(self, series_type, vals_type): + N_many = 10**5 + + if series_type == "nans": + ser_vals = np.full(10**4, np.nan) + elif series_type == "short": + ser_vals = np.arange(2) + elif series_type == "long": + ser_vals = np.arange(N_many) + elif series_type == "long_floats": + ser_vals = np.arange(N_many, dtype=np.float64) + + self.series = Series(ser_vals).astype(object) + + if vals_type == "nans": + values = np.full(10**4, np.nan) + elif vals_type == "short": + values = np.arange(2) + elif vals_type == "long": + values = np.arange(N_many) + elif vals_type == "long_floats": + values = np.arange(N_many, dtype=np.float64) + + self.values = values.astype(object) + + def time_isin(self, series_type, vals_type): + self.series.isin(self.values) + + +class IsInLongSeriesLookUpDominates: + params = [ + ["int64", "int32", "float64", "float32", "object", "Int64", "Float64"], + [5, 1000], + ["random_hits", "random_misses", "monotone_hits", "monotone_misses"], + ] + param_names = ["dtype", "MaxNumber", "series_type"] + + def setup(self, dtype, MaxNumber, series_type): + N = 10**7 + + if series_type == "random_hits": + array = np.random.randint(0, MaxNumber, N) + if series_type == "random_misses": + array = np.random.randint(0, MaxNumber, N) + MaxNumber + if series_type == "monotone_hits": + array = np.repeat(np.arange(MaxNumber), N // MaxNumber) + if series_type == "monotone_misses": + array = np.arange(N) + MaxNumber + + self.series = Series(array).astype(dtype) + + self.values = np.arange(MaxNumber).astype(dtype.lower()) + + def time_isin(self, dtypes, MaxNumber, series_type): + self.series.isin(self.values) + + +class IsInLongSeriesValuesDominate: + params = [ + ["int64", "int32", "float64", "float32", "object", "Int64", "Float64"], + ["random", "monotone"], + ] + param_names = ["dtype", "series_type"] + + def setup(self, dtype, series_type): + N = 10**7 + + if series_type == "random": + vals = np.random.randint(0, 10 * N, N) + if series_type == "monotone": + vals = np.arange(N) + + self.values = vals.astype(dtype.lower()) + M = 10**6 + 1 + self.series = Series(np.arange(M)).astype(dtype) + + def time_isin(self, dtypes, series_type): + self.series.isin(self.values) + + +class IsInWithLongTupples: + def setup(self): + t = tuple(range(1000)) + self.series = Series([t] * 1000) + self.values = [t] + + def time_isin(self): + self.series.isin(self.values) + + +class IsInIndexes: + def setup(self): + self.range_idx = Index(range(1000)) + self.index = Index(list(range(1000))) + self.series = Series(np.random.randint(100_000, size=1000)) + + def time_isin_range_index(self): + self.series.isin(self.range_idx) + + def time_isin_index(self): + self.series.isin(self.index) diff --git a/asv_bench/benchmarks/arithmetic.py b/asv_bench/benchmarks/arithmetic.py new file mode 100644 index 0000000000000..6b1f75187f887 --- /dev/null +++ b/asv_bench/benchmarks/arithmetic.py @@ -0,0 +1,479 @@ +import operator +import warnings + +import numpy as np + +import pandas as pd +from pandas import ( + DataFrame, + Index, + Series, + Timestamp, + date_range, + to_timedelta, +) + +from .pandas_vb_common import numeric_dtypes + +try: + import pandas.core.computation.expressions as expr +except ImportError: + import pandas.computation.expressions as expr +try: + import pandas.tseries.holiday +except ImportError: + pass + + +class IntFrameWithScalar: + params = [ + [np.float64, np.int64], + [2, 3.0, np.int32(4), np.float64(5)], + [ + operator.add, + operator.sub, + operator.mul, + operator.truediv, + operator.floordiv, + operator.pow, + operator.mod, + operator.eq, + operator.ne, + operator.gt, + operator.ge, + operator.lt, + operator.le, + ], + ] + param_names = ["dtype", "scalar", "op"] + + def setup(self, dtype, scalar, op): + arr = np.random.randn(20000, 100) + self.df = DataFrame(arr.astype(dtype)) + + def time_frame_op_with_scalar(self, dtype, scalar, op): + op(self.df, scalar) + + +class OpWithFillValue: + def setup(self): + # GH#31300 + arr = np.arange(10**6) + df = DataFrame({"A": arr}) + ser = df["A"] + + self.df = df + self.ser = ser + + def time_frame_op_with_fill_value_no_nas(self): + self.df.add(self.df, fill_value=4) + + def time_series_op_with_fill_value_no_nas(self): + self.ser.add(self.ser, fill_value=4) + + +class MixedFrameWithSeriesAxis: + params = [ + [ + "eq", + "ne", + "lt", + "le", + "ge", + "gt", + "add", + "sub", + "truediv", + "floordiv", + "mul", + "pow", + ] + ] + param_names = ["opname"] + + def setup(self, opname): + arr = np.arange(10**6).reshape(1000, -1) + df = DataFrame(arr) + df["C"] = 1.0 + self.df = df + self.ser = df[0] + self.row = df.iloc[0] + + def time_frame_op_with_series_axis0(self, opname): + getattr(self.df, opname)(self.ser, axis=0) + + def time_frame_op_with_series_axis1(self, opname): + getattr(operator, opname)(self.df, self.ser) + + # exclude comparisons from the params for time_frame_op_with_series_axis1 + # since they do not do alignment so raise + time_frame_op_with_series_axis1.params = [params[0][6:]] + + +class FrameWithFrameWide: + # Many-columns, mixed dtypes + + params = [ + [ + # GH#32779 has discussion of which operators are included here + operator.add, + operator.floordiv, + operator.gt, + ], + [ + # (n_rows, n_columns) + (1_000_000, 10), + (100_000, 100), + (10_000, 1000), + (1000, 10_000), + ], + ] + param_names = ["op", "shape"] + + def setup(self, op, shape): + # we choose dtypes so as to make the blocks + # a) not perfectly match between right and left + # b) appreciably bigger than single columns + n_rows, n_cols = shape + + if op is operator.floordiv: + # floordiv is much slower than the other operations -> use less data + n_rows = n_rows // 10 + + # construct dataframe with 2 blocks + arr1 = np.random.randn(n_rows, n_cols // 2).astype("f8") + arr2 = np.random.randn(n_rows, n_cols // 2).astype("f4") + df = pd.concat([DataFrame(arr1), DataFrame(arr2)], axis=1, ignore_index=True) + # should already be the case, but just to be sure + df._consolidate_inplace() + + # TODO: GH#33198 the setting here shouldn't need two steps + arr1 = np.random.randn(n_rows, max(n_cols // 4, 3)).astype("f8") + arr2 = np.random.randn(n_rows, n_cols // 2).astype("i8") + arr3 = np.random.randn(n_rows, n_cols // 4).astype("f8") + df2 = pd.concat( + [DataFrame(arr1), DataFrame(arr2), DataFrame(arr3)], + axis=1, + ignore_index=True, + ) + # should already be the case, but just to be sure + df2._consolidate_inplace() + + self.left = df + self.right = df2 + + def time_op_different_blocks(self, op, shape): + # blocks (and dtypes) are not aligned + op(self.left, self.right) + + def time_op_same_blocks(self, op, shape): + # blocks (and dtypes) are aligned + op(self.left, self.left) + + +class Ops: + params = [[True, False], ["default", 1]] + param_names = ["use_numexpr", "threads"] + + def setup(self, use_numexpr, threads): + self.df = DataFrame(np.random.randn(20000, 100)) + self.df2 = DataFrame(np.random.randn(20000, 100)) + + if threads != "default": + expr.set_numexpr_threads(threads) + if not use_numexpr: + expr.set_use_numexpr(False) + + def time_frame_add(self, use_numexpr, threads): + self.df + self.df2 + + def time_frame_mult(self, use_numexpr, threads): + self.df * self.df2 + + def time_frame_multi_and(self, use_numexpr, threads): + self.df[(self.df > 0) & (self.df2 > 0)] + + def time_frame_comparison(self, use_numexpr, threads): + self.df > self.df2 + + def teardown(self, use_numexpr, threads): + expr.set_use_numexpr(True) + expr.set_numexpr_threads() + + +class Ops2: + def setup(self): + N = 10**3 + self.df = DataFrame(np.random.randn(N, N)) + self.df2 = DataFrame(np.random.randn(N, N)) + + self.df_int = DataFrame( + np.random.randint( + np.iinfo(np.int16).min, np.iinfo(np.int16).max, size=(N, N) + ) + ) + self.df2_int = DataFrame( + np.random.randint( + np.iinfo(np.int16).min, np.iinfo(np.int16).max, size=(N, N) + ) + ) + + self.s = Series(np.random.randn(N)) + + # Division + + def time_frame_float_div(self): + self.df // self.df2 + + def time_frame_float_div_by_zero(self): + self.df / 0 + + def time_frame_float_floor_by_zero(self): + self.df // 0 + + def time_frame_int_div_by_zero(self): + self.df_int / 0 + + # Modulo + + def time_frame_int_mod(self): + self.df_int % self.df2_int + + def time_frame_float_mod(self): + self.df % self.df2 + + # Dot product + + def time_frame_dot(self): + self.df.dot(self.df2) + + def time_series_dot(self): + self.s.dot(self.s) + + def time_frame_series_dot(self): + self.df.dot(self.s) + + +class Timeseries: + params = [None, "US/Eastern"] + param_names = ["tz"] + + def setup(self, tz): + N = 10**6 + halfway = (N // 2) - 1 + self.s = Series(date_range("20010101", periods=N, freq="min", tz=tz)) + self.ts = self.s[halfway] + + self.s2 = Series(date_range("20010101", periods=N, freq="s", tz=tz)) + self.ts_different_reso = Timestamp("2001-01-02", tz=tz) + + def time_series_timestamp_compare(self, tz): + self.s <= self.ts + + def time_series_timestamp_different_reso_compare(self, tz): + self.s <= self.ts_different_reso + + def time_timestamp_series_compare(self, tz): + self.ts >= self.s + + def time_timestamp_ops_diff(self, tz): + self.s2.diff() + + def time_timestamp_ops_diff_with_shift(self, tz): + self.s - self.s.shift() + + +class IrregularOps: + def setup(self): + N = 10**5 + idx = date_range(start="1/1/2000", periods=N, freq="s") + s = Series(np.random.randn(N), index=idx) + self.left = s.sample(frac=1) + self.right = s.sample(frac=1) + + def time_add(self): + self.left + self.right + + +class TimedeltaOps: + def setup(self): + self.td = to_timedelta(np.arange(1000000)) + self.ts = Timestamp("2000") + + def time_add_td_ts(self): + self.td + self.ts + + +class CategoricalComparisons: + params = ["__lt__", "__le__", "__eq__", "__ne__", "__ge__", "__gt__"] + param_names = ["op"] + + def setup(self, op): + N = 10**5 + self.cat = pd.Categorical(list("aabbcd") * N, ordered=True) + + def time_categorical_op(self, op): + getattr(self.cat, op)("b") + + +class IndexArithmetic: + params = ["float", "int"] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10**6 + if dtype == "float": + self.index = Index(np.arange(N), dtype=np.float64) + elif dtype == "int": + self.index = Index(np.arange(N), dtype=np.int64) + + def time_add(self, dtype): + self.index + 2 + + def time_subtract(self, dtype): + self.index - 2 + + def time_multiply(self, dtype): + self.index * 2 + + def time_divide(self, dtype): + self.index / 2 + + def time_modulo(self, dtype): + self.index % 2 + + +class NumericInferOps: + # from GH 7332 + params = numeric_dtypes + param_names = ["dtype"] + + def setup(self, dtype): + N = 5 * 10**5 + self.df = DataFrame( + {"A": np.arange(N).astype(dtype), "B": np.arange(N).astype(dtype)} + ) + + def time_add(self, dtype): + self.df["A"] + self.df["B"] + + def time_subtract(self, dtype): + self.df["A"] - self.df["B"] + + def time_multiply(self, dtype): + self.df["A"] * self.df["B"] + + def time_divide(self, dtype): + self.df["A"] / self.df["B"] + + def time_modulo(self, dtype): + self.df["A"] % self.df["B"] + + +class DateInferOps: + # from GH 7332 + def setup_cache(self): + N = 5 * 10**5 + df = DataFrame({"datetime64": np.arange(N).astype("datetime64[ms]")}) + df["timedelta"] = df["datetime64"] - df["datetime64"] + return df + + def time_subtract_datetimes(self, df): + df["datetime64"] - df["datetime64"] + + def time_timedelta_plus_datetime(self, df): + df["timedelta"] + df["datetime64"] + + def time_add_timedeltas(self, df): + df["timedelta"] + df["timedelta"] + + +hcal = pd.tseries.holiday.USFederalHolidayCalendar() +# These offsets currently raise a NotImplementedError with .apply_index() +non_apply = [ + pd.offsets.Day(), + pd.offsets.BYearEnd(), + pd.offsets.BYearBegin(), + pd.offsets.BQuarterEnd(), + pd.offsets.BQuarterBegin(), + pd.offsets.BMonthEnd(), + pd.offsets.BMonthBegin(), + pd.offsets.CustomBusinessDay(), + pd.offsets.CustomBusinessDay(calendar=hcal), + pd.offsets.CustomBusinessMonthBegin(calendar=hcal), + pd.offsets.CustomBusinessMonthEnd(calendar=hcal), + pd.offsets.CustomBusinessMonthEnd(calendar=hcal), +] +other_offsets = [ + pd.offsets.YearEnd(), + pd.offsets.YearBegin(), + pd.offsets.QuarterEnd(), + pd.offsets.QuarterBegin(), + pd.offsets.MonthEnd(), + pd.offsets.MonthBegin(), + pd.offsets.DateOffset(months=2, days=2), + pd.offsets.BusinessDay(), + pd.offsets.SemiMonthEnd(), + pd.offsets.SemiMonthBegin(), +] +offsets = non_apply + other_offsets + + +class OffsetArrayArithmetic: + params = offsets + param_names = ["offset"] + + def setup(self, offset): + N = 10000 + rng = date_range(start="1/1/2000", periods=N, freq="min") + self.rng = rng + self.ser = Series(rng) + + def time_add_series_offset(self, offset): + with warnings.catch_warnings(record=True): + self.ser + offset + + def time_add_dti_offset(self, offset): + with warnings.catch_warnings(record=True): + self.rng + offset + + +class ApplyIndex: + params = other_offsets + param_names = ["offset"] + + def setup(self, offset): + N = 10000 + rng = date_range(start="1/1/2000", periods=N, freq="min") + self.rng = rng + + def time_apply_index(self, offset): + self.rng + offset + + +class BinaryOpsMultiIndex: + params = ["sub", "add", "mul", "div"] + param_names = ["func"] + + def setup(self, func): + array = date_range("20200101 00:00", "20200102 0:00", freq="s") + level_0_names = [str(i) for i in range(30)] + + index = pd.MultiIndex.from_product([level_0_names, array]) + column_names = ["col_1", "col_2"] + + self.df = DataFrame( + np.random.rand(len(index), 2), index=index, columns=column_names + ) + + self.arg_df = DataFrame( + np.random.randint(1, 10, (len(level_0_names), 2)), + index=level_0_names, + columns=column_names, + ) + + def time_binary_op_multiindex(self, func): + getattr(self.df, func)(self.arg_df, level=0) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/array.py b/asv_bench/benchmarks/array.py new file mode 100644 index 0000000000000..953af5c868356 --- /dev/null +++ b/asv_bench/benchmarks/array.py @@ -0,0 +1,140 @@ +import numpy as np + +import pandas as pd + + +class BooleanArray: + def setup(self): + self.values_bool = np.array([True, False, True, False]) + self.values_float = np.array([1.0, 0.0, 1.0, 0.0]) + self.values_integer = np.array([1, 0, 1, 0]) + self.values_integer_like = [1, 0, 1, 0] + self.data = np.array([True, False, True, False]) + self.mask = np.array([False, False, True, False]) + + def time_constructor(self): + pd.arrays.BooleanArray(self.data, self.mask) + + def time_from_bool_array(self): + pd.array(self.values_bool, dtype="boolean") + + def time_from_integer_array(self): + pd.array(self.values_integer, dtype="boolean") + + def time_from_integer_like(self): + pd.array(self.values_integer_like, dtype="boolean") + + def time_from_float_array(self): + pd.array(self.values_float, dtype="boolean") + + +class IntegerArray: + def setup(self): + N = 250_000 + self.values_integer = np.tile(np.array([1, 0, 1, 0]), N) + self.data = np.tile(np.array([1, 2, 3, 4], dtype="int64"), N) + self.mask = np.tile(np.array([False, False, True, False]), N) + + def time_constructor(self): + pd.arrays.IntegerArray(self.data, self.mask) + + def time_from_integer_array(self): + pd.array(self.values_integer, dtype="Int64") + + +class IntervalArray: + def setup(self): + N = 10_000 + self.tuples = [(i, i + 1) for i in range(N)] + + def time_from_tuples(self): + pd.arrays.IntervalArray.from_tuples(self.tuples) + + +class StringArray: + def setup(self): + N = 100_000 + values = np.array([str(i) for i in range(N)], dtype=object) + self.values_obj = np.array(values, dtype="object") + self.values_str = np.array(values, dtype="U") + self.values_list = values.tolist() + + def time_from_np_object_array(self): + pd.array(self.values_obj, dtype="string") + + def time_from_np_str_array(self): + pd.array(self.values_str, dtype="string") + + def time_from_list(self): + pd.array(self.values_list, dtype="string") + + +class ArrowStringArray: + params = [False, True] + param_names = ["multiple_chunks"] + + def setup(self, multiple_chunks): + try: + import pyarrow as pa + except ImportError as err: + raise NotImplementedError from err + strings = np.array([str(i) for i in range(10_000)], dtype=object) + if multiple_chunks: + chunks = [strings[i : i + 100] for i in range(0, len(strings), 100)] + self.array = pd.arrays.ArrowStringArray(pa.chunked_array(chunks)) + else: + self.array = pd.arrays.ArrowStringArray(pa.array(strings)) + + def time_setitem(self, multiple_chunks): + for i in range(200): + self.array[i] = "foo" + + def time_setitem_list(self, multiple_chunks): + indexer = list(range(50)) + list(range(-1000, 0, 50)) + self.array[indexer] = ["foo"] * len(indexer) + + def time_setitem_slice(self, multiple_chunks): + self.array[::10] = "foo" + + def time_setitem_null_slice(self, multiple_chunks): + self.array[:] = "foo" + + def time_tolist(self, multiple_chunks): + self.array.tolist() + + +class ArrowExtensionArray: + params = [ + [ + "boolean[pyarrow]", + "float64[pyarrow]", + "int64[pyarrow]", + "string[pyarrow]", + "timestamp[ns][pyarrow]", + ], + [False, True], + ] + param_names = ["dtype", "hasna"] + + def setup(self, dtype, hasna): + N = 100_000 + if dtype == "boolean[pyarrow]": + data = np.random.choice([True, False], N, replace=True) + elif dtype == "float64[pyarrow]": + data = np.random.randn(N) + elif dtype == "int64[pyarrow]": + data = np.arange(N) + elif dtype == "string[pyarrow]": + data = np.array([str(i) for i in range(N)], dtype=object) + elif dtype == "timestamp[ns][pyarrow]": + data = pd.date_range("2000-01-01", freq="s", periods=N) + else: + raise NotImplementedError + + arr = pd.array(data, dtype=dtype) + if hasna: + arr[::2] = pd.NA + self.arr = arr + + def time_to_numpy(self, dtype, hasna): + self.arr.to_numpy() diff --git a/asv_bench/benchmarks/attrs_caching.py b/asv_bench/benchmarks/attrs_caching.py new file mode 100644 index 0000000000000..2a004113d1b91 --- /dev/null +++ b/asv_bench/benchmarks/attrs_caching.py @@ -0,0 +1,50 @@ +import numpy as np + +import pandas as pd +from pandas import DataFrame + +try: + from pandas.core.construction import extract_array +except ImportError: + extract_array = None + + +class DataFrameAttributes: + def setup(self): + self.df = DataFrame(np.random.randn(10, 6)) + self.cur_index = self.df.index + + def time_get_index(self): + self.df.index + + def time_set_index(self): + self.df.index = self.cur_index + + +class SeriesArrayAttribute: + params = [["numeric", "object", "category", "datetime64", "datetime64tz"]] + param_names = ["dtype"] + + def setup(self, dtype): + if dtype == "numeric": + self.series = pd.Series([1, 2, 3]) + elif dtype == "object": + self.series = pd.Series(["a", "b", "c"], dtype=object) + elif dtype == "category": + self.series = pd.Series(["a", "b", "c"], dtype="category") + elif dtype == "datetime64": + self.series = pd.Series(pd.date_range("2013", periods=3)) + elif dtype == "datetime64tz": + self.series = pd.Series(pd.date_range("2013", periods=3, tz="UTC")) + + def time_array(self, dtype): + self.series.array + + def time_extract_array(self, dtype): + extract_array(self.series) + + def time_extract_array_numpy(self, dtype): + extract_array(self.series, extract_numpy=True) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/boolean.py b/asv_bench/benchmarks/boolean.py new file mode 100644 index 0000000000000..71c422c641775 --- /dev/null +++ b/asv_bench/benchmarks/boolean.py @@ -0,0 +1,32 @@ +import numpy as np + +import pandas as pd + + +class TimeLogicalOps: + def setup(self): + N = 10_000 + left, right, lmask, rmask = np.random.randint(0, 2, size=(4, N)).astype("bool") + self.left = pd.arrays.BooleanArray(left, lmask) + self.right = pd.arrays.BooleanArray(right, rmask) + + def time_or_scalar(self): + self.left | True + self.left | False + + def time_or_array(self): + self.left | self.right + + def time_and_scalar(self): + self.left & True + self.left & False + + def time_and_array(self): + self.left & self.right + + def time_xor_scalar(self): + self.left ^ True + self.left ^ False + + def time_xor_array(self): + self.left ^ self.right diff --git a/asv_bench/benchmarks/categoricals.py b/asv_bench/benchmarks/categoricals.py new file mode 100644 index 0000000000000..7d5b250c7b157 --- /dev/null +++ b/asv_bench/benchmarks/categoricals.py @@ -0,0 +1,337 @@ +import string +import sys +import warnings + +import numpy as np + +import pandas as pd + +try: + from pandas.api.types import union_categoricals +except ImportError: + try: + from pandas.types.concat import union_categoricals + except ImportError: + pass + + +class Constructor: + def setup(self): + N = 10**5 + self.categories = list("abcde") + self.cat_idx = pd.Index(self.categories) + self.values = np.tile(self.categories, N) + self.codes = np.tile(range(len(self.categories)), N) + + self.datetimes = pd.Series( + pd.date_range("1995-01-01 00:00:00", periods=N // 10, freq="s") + ) + self.datetimes_with_nat = self.datetimes.copy() + self.datetimes_with_nat.iloc[-1] = pd.NaT + + self.values_some_nan = list(np.tile(self.categories + [np.nan], N)) + self.values_all_nan = [np.nan] * len(self.values) + self.values_all_int8 = np.ones(N, "int8") + self.categorical = pd.Categorical(self.values, self.categories) + self.series = pd.Series(self.categorical) + self.intervals = pd.interval_range(0, 1, periods=N // 10) + + def time_regular(self): + pd.Categorical(self.values, self.categories) + + def time_fastpath(self): + dtype = pd.CategoricalDtype(categories=self.cat_idx) + pd.Categorical._simple_new(self.codes, dtype) + + def time_datetimes(self): + pd.Categorical(self.datetimes) + + def time_interval(self): + pd.Categorical(self.datetimes, categories=self.datetimes) + + def time_datetimes_with_nat(self): + pd.Categorical(self.datetimes_with_nat) + + def time_with_nan(self): + pd.Categorical(self.values_some_nan) + + def time_all_nan(self): + pd.Categorical(self.values_all_nan) + + def time_from_codes_all_int8(self): + pd.Categorical.from_codes(self.values_all_int8, self.categories) + + def time_existing_categorical(self): + pd.Categorical(self.categorical) + + def time_existing_series(self): + pd.Categorical(self.series) + + +class AsType: + def setup(self): + N = 10**5 + + random_pick = np.random.default_rng().choice + + categories = { + "str": list(string.ascii_letters), + "int": np.random.randint(2**16, size=154), + "float": sys.maxsize * np.random.random((38,)), + "timestamp": [ + pd.Timestamp(x, unit="s") for x in np.random.randint(2**18, size=578) + ], + } + + self.df = pd.DataFrame( + {col: random_pick(cats, N) for col, cats in categories.items()} + ) + + for col in ("int", "float", "timestamp"): + self.df[f"{col}_as_str"] = self.df[col].astype(str) + + for col in self.df.columns: + self.df[col] = self.df[col].astype("category") + + def astype_str(self): + [self.df[col].astype("str") for col in "int float timestamp".split()] + + def astype_int(self): + [self.df[col].astype("int") for col in "int_as_str timestamp".split()] + + def astype_float(self): + [ + self.df[col].astype("float") + for col in "float_as_str int int_as_str timestamp".split() + ] + + def astype_datetime(self): + self.df["float"].astype(pd.DatetimeTZDtype(tz="US/Pacific")) + + +class Concat: + def setup(self): + N = 10**5 + self.s = pd.Series(list("aabbcd") * N).astype("category") + + self.a = pd.Categorical(list("aabbcd") * N) + self.b = pd.Categorical(list("bbcdjk") * N) + + self.idx_a = pd.CategoricalIndex(range(N), range(N)) + self.idx_b = pd.CategoricalIndex(range(N + 1), range(N + 1)) + self.df_a = pd.DataFrame(range(N), columns=["a"], index=self.idx_a) + self.df_b = pd.DataFrame(range(N + 1), columns=["a"], index=self.idx_b) + + def time_concat(self): + pd.concat([self.s, self.s]) + + def time_union(self): + union_categoricals([self.a, self.b]) + + def time_append_overlapping_index(self): + self.idx_a.append(self.idx_a) + + def time_append_non_overlapping_index(self): + self.idx_a.append(self.idx_b) + + def time_concat_overlapping_index(self): + pd.concat([self.df_a, self.df_a]) + + def time_concat_non_overlapping_index(self): + pd.concat([self.df_a, self.df_b]) + + +class ValueCounts: + params = [True, False] + param_names = ["dropna"] + + def setup(self, dropna): + n = 5 * 10**5 + arr = [f"s{i:04d}" for i in np.random.randint(0, n // 10, size=n)] + self.ts = pd.Series(arr).astype("category") + + def time_value_counts(self, dropna): + self.ts.value_counts(dropna=dropna) + + +class Repr: + def setup(self): + self.sel = pd.Series(["s1234"]).astype("category") + + def time_rendering(self): + str(self.sel) + + +class SetCategories: + def setup(self): + n = 5 * 10**5 + arr = [f"s{i:04d}" for i in np.random.randint(0, n // 10, size=n)] + self.ts = pd.Series(arr).astype("category") + + def time_set_categories(self): + self.ts.cat.set_categories(self.ts.cat.categories[::2]) + + +class RemoveCategories: + def setup(self): + n = 5 * 10**5 + arr = [f"s{i:04d}" for i in np.random.randint(0, n // 10, size=n)] + self.ts = pd.Series(arr).astype("category") + + def time_remove_categories(self): + self.ts.cat.remove_categories(self.ts.cat.categories[::2]) + + +class Rank: + def setup(self): + N = 10**5 + ncats = 15 + + self.s_str = pd.Series(np.random.randint(0, ncats, size=N).astype(str)) + self.s_str_cat = pd.Series(self.s_str, dtype="category") + with warnings.catch_warnings(record=True): + str_cat_type = pd.CategoricalDtype(set(self.s_str), ordered=True) + self.s_str_cat_ordered = self.s_str.astype(str_cat_type) + + self.s_int = pd.Series(np.random.randint(0, ncats, size=N)) + self.s_int_cat = pd.Series(self.s_int, dtype="category") + with warnings.catch_warnings(record=True): + int_cat_type = pd.CategoricalDtype(set(self.s_int), ordered=True) + self.s_int_cat_ordered = self.s_int.astype(int_cat_type) + + def time_rank_string(self): + self.s_str.rank() + + def time_rank_string_cat(self): + self.s_str_cat.rank() + + def time_rank_string_cat_ordered(self): + self.s_str_cat_ordered.rank() + + def time_rank_int(self): + self.s_int.rank() + + def time_rank_int_cat(self): + self.s_int_cat.rank() + + def time_rank_int_cat_ordered(self): + self.s_int_cat_ordered.rank() + + +class IsMonotonic: + def setup(self): + N = 1000 + self.c = pd.CategoricalIndex(list("a" * N + "b" * N + "c" * N)) + self.s = pd.Series(self.c) + + def time_categorical_index_is_monotonic_increasing(self): + self.c.is_monotonic_increasing + + def time_categorical_index_is_monotonic_decreasing(self): + self.c.is_monotonic_decreasing + + def time_categorical_series_is_monotonic_increasing(self): + self.s.is_monotonic_increasing + + def time_categorical_series_is_monotonic_decreasing(self): + self.s.is_monotonic_decreasing + + +class Contains: + def setup(self): + N = 10**5 + self.ci = pd.CategoricalIndex(np.arange(N)) + self.c = self.ci.values + self.key = self.ci.categories[0] + + def time_categorical_index_contains(self): + self.key in self.ci + + def time_categorical_contains(self): + self.key in self.c + + +class CategoricalSlicing: + params = ["monotonic_incr", "monotonic_decr", "non_monotonic"] + param_names = ["index"] + + def setup(self, index): + N = 10**6 + categories = ["a", "b", "c"] + if index == "monotonic_incr": + codes = np.repeat([0, 1, 2], N) + elif index == "monotonic_decr": + codes = np.repeat([2, 1, 0], N) + elif index == "non_monotonic": + codes = np.tile([0, 1, 2], N) + else: + raise ValueError(f"Invalid index param: {index}") + + self.data = pd.Categorical.from_codes(codes, categories=categories) + self.scalar = 10000 + self.list = list(range(10000)) + self.cat_scalar = "b" + + def time_getitem_scalar(self, index): + self.data[self.scalar] + + def time_getitem_slice(self, index): + self.data[: self.scalar] + + def time_getitem_list_like(self, index): + self.data[[self.scalar]] + + def time_getitem_list(self, index): + self.data[self.list] + + def time_getitem_bool_array(self, index): + self.data[self.data == self.cat_scalar] + + +class Indexing: + def setup(self): + N = 10**5 + self.index = pd.CategoricalIndex(range(N), range(N)) + self.series = pd.Series(range(N), index=self.index).sort_index() + self.category = self.index[500] + + def time_get_loc(self): + self.index.get_loc(self.category) + + def time_shallow_copy(self): + self.index._view() + + def time_align(self): + pd.DataFrame({"a": self.series, "b": self.series[:500]}) + + def time_intersection(self): + self.index[:750].intersection(self.index[250:]) + + def time_unique(self): + self.index.unique() + + def time_reindex(self): + self.index.reindex(self.index[:500]) + + def time_reindex_missing(self): + self.index.reindex(["a", "b", "c", "d"]) + + def time_sort_values(self): + self.index.sort_values(ascending=False) + + +class SearchSorted: + def setup(self): + N = 10**5 + self.ci = pd.CategoricalIndex(np.arange(N)).sort_values() + self.c = self.ci.values + self.key = self.ci.categories[1] + + def time_categorical_index_contains(self): + self.ci.searchsorted(self.key) + + def time_categorical_contains(self): + self.c.searchsorted(self.key) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/ctors.py b/asv_bench/benchmarks/ctors.py new file mode 100644 index 0000000000000..77c9faf3d3a87 --- /dev/null +++ b/asv_bench/benchmarks/ctors.py @@ -0,0 +1,145 @@ +import numpy as np + +from pandas import ( + DatetimeIndex, + Index, + MultiIndex, + Series, + Timestamp, + date_range, +) + + +def no_change(arr): + return arr + + +def list_of_str(arr): + return list(arr.astype(str)) + + +def gen_of_str(arr): + return (x for x in arr.astype(str)) + + +def arr_dict(arr): + return dict(zip(range(len(arr)), arr)) + + +def list_of_tuples(arr): + return [(i, -i) for i in arr] + + +def gen_of_tuples(arr): + return ((i, -i) for i in arr) + + +def list_of_lists(arr): + return [[i, -i] for i in arr] + + +def list_of_tuples_with_none(arr): + return [(i, -i) for i in arr][:-1] + [None] + + +def list_of_lists_with_none(arr): + return [[i, -i] for i in arr][:-1] + [None] + + +class SeriesConstructors: + param_names = ["data_fmt", "with_index", "dtype"] + params = [ + [ + no_change, + list, + list_of_str, + gen_of_str, + arr_dict, + list_of_tuples, + gen_of_tuples, + list_of_lists, + list_of_tuples_with_none, + list_of_lists_with_none, + ], + [False, True], + ["float", "int"], + ] + + # Generators get exhausted on use, so run setup before every call + number = 1 + repeat = (3, 250, 10) + + def setup(self, data_fmt, with_index, dtype): + if data_fmt in (gen_of_str, gen_of_tuples) and with_index: + raise NotImplementedError( + "Series constructors do not support using generators with indexes" + ) + N = 10**4 + if dtype == "float": + arr = np.random.randn(N) + else: + arr = np.arange(N) + self.data = data_fmt(arr) + self.index = np.arange(N) if with_index else None + + def time_series_constructor(self, data_fmt, with_index, dtype): + Series(self.data, index=self.index) + + +class SeriesDtypesConstructors: + def setup(self): + N = 10**4 + self.arr = np.random.randn(N) + self.arr_str = np.array(["foo", "bar", "baz"], dtype=object) + self.s = Series( + [Timestamp("20110101"), Timestamp("20120101"), Timestamp("20130101")] + * N + * 10 + ) + + def time_index_from_array_string(self): + Index(self.arr_str) + + def time_index_from_array_floats(self): + Index(self.arr) + + def time_dtindex_from_series(self): + DatetimeIndex(self.s) + + def time_dtindex_from_index_with_series(self): + Index(self.s) + + +class MultiIndexConstructor: + def setup(self): + N = 10**4 + self.iterables = [Index([f"i-{i}" for i in range(N)], dtype=object), range(20)] + + def time_multiindex_from_iterables(self): + MultiIndex.from_product(self.iterables) + + +class DatetimeIndexConstructor: + def setup(self): + N = 20_000 + dti = date_range("1900-01-01", periods=N) + + self.list_of_timestamps = dti.tolist() + self.list_of_dates = dti.date.tolist() + self.list_of_datetimes = dti.to_pydatetime().tolist() + self.list_of_str = dti.strftime("%Y-%m-%d").tolist() + + def time_from_list_of_timestamps(self): + DatetimeIndex(self.list_of_timestamps) + + def time_from_list_of_dates(self): + DatetimeIndex(self.list_of_dates) + + def time_from_list_of_datetimes(self): + DatetimeIndex(self.list_of_datetimes) + + def time_from_list_of_str(self): + DatetimeIndex(self.list_of_str) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/dtypes.py b/asv_bench/benchmarks/dtypes.py new file mode 100644 index 0000000000000..7f3429b5e3882 --- /dev/null +++ b/asv_bench/benchmarks/dtypes.py @@ -0,0 +1,129 @@ +import string + +import numpy as np + +import pandas as pd +from pandas import ( + DataFrame, + Index, +) +import pandas._testing as tm +from pandas.api.types import ( + is_extension_array_dtype, + pandas_dtype, +) + +from .pandas_vb_common import ( + datetime_dtypes, + extension_dtypes, + numeric_dtypes, + string_dtypes, +) + +_numpy_dtypes = [ + np.dtype(dtype) for dtype in (numeric_dtypes + datetime_dtypes + string_dtypes) +] +_dtypes = _numpy_dtypes + extension_dtypes + + +class Dtypes: + params = _dtypes + [dt.name for dt in _dtypes] + param_names = ["dtype"] + + def time_pandas_dtype(self, dtype): + pandas_dtype(dtype) + + +class DtypesInvalid: + param_names = ["dtype"] + params = ["scalar-string", "scalar-int", "list-string", "array-string"] + data_dict = { + "scalar-string": "foo", + "scalar-int": 1, + "list-string": ["foo"] * 1000, + "array-string": np.array(["foo"] * 1000), + } + + def time_pandas_dtype_invalid(self, dtype): + try: + pandas_dtype(self.data_dict[dtype]) + except TypeError: + pass + + +class SelectDtypes: + try: + params = [ + tm.ALL_INT_NUMPY_DTYPES + + tm.ALL_INT_EA_DTYPES + + tm.FLOAT_NUMPY_DTYPES + + tm.COMPLEX_DTYPES + + tm.DATETIME64_DTYPES + + tm.TIMEDELTA64_DTYPES + + tm.BOOL_DTYPES + ] + except AttributeError: + params = [ + tm.ALL_INT_DTYPES + + tm.ALL_EA_INT_DTYPES + + tm.FLOAT_DTYPES + + tm.COMPLEX_DTYPES + + tm.DATETIME64_DTYPES + + tm.TIMEDELTA64_DTYPES + + tm.BOOL_DTYPES + ] + param_names = ["dtype"] + + def setup(self, dtype): + N, K = 5000, 50 + self.index = Index([f"i-{i}" for i in range(N)], dtype=object) + self.columns = Index([f"i-{i}" for i in range(K)], dtype=object) + + def create_df(data): + return DataFrame(data, index=self.index, columns=self.columns) + + self.df_int = create_df(np.random.randint(low=100, size=(N, K))) + self.df_float = create_df(np.random.randn(N, K)) + self.df_bool = create_df(np.random.choice([True, False], size=(N, K))) + self.df_string = create_df( + np.random.choice(list(string.ascii_letters), size=(N, K)) + ) + + def time_select_dtype_int_include(self, dtype): + self.df_int.select_dtypes(include=dtype) + + def time_select_dtype_int_exclude(self, dtype): + self.df_int.select_dtypes(exclude=dtype) + + def time_select_dtype_float_include(self, dtype): + self.df_float.select_dtypes(include=dtype) + + def time_select_dtype_float_exclude(self, dtype): + self.df_float.select_dtypes(exclude=dtype) + + def time_select_dtype_bool_include(self, dtype): + self.df_bool.select_dtypes(include=dtype) + + def time_select_dtype_bool_exclude(self, dtype): + self.df_bool.select_dtypes(exclude=dtype) + + def time_select_dtype_string_include(self, dtype): + self.df_string.select_dtypes(include=dtype) + + def time_select_dtype_string_exclude(self, dtype): + self.df_string.select_dtypes(exclude=dtype) + + +class CheckDtypes: + def setup(self): + self.ext_dtype = pd.Int64Dtype() + self.np_dtype = np.dtype("int64") + + def time_is_extension_array_dtype_true(self): + is_extension_array_dtype(self.ext_dtype) + + def time_is_extension_array_dtype_false(self): + is_extension_array_dtype(self.np_dtype) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/eval.py b/asv_bench/benchmarks/eval.py new file mode 100644 index 0000000000000..656d16a910a9f --- /dev/null +++ b/asv_bench/benchmarks/eval.py @@ -0,0 +1,65 @@ +import numpy as np + +import pandas as pd + +try: + import pandas.core.computation.expressions as expr +except ImportError: + import pandas.computation.expressions as expr + + +class Eval: + params = [["numexpr", "python"], [1, "all"]] + param_names = ["engine", "threads"] + + def setup(self, engine, threads): + self.df = pd.DataFrame(np.random.randn(20000, 100)) + self.df2 = pd.DataFrame(np.random.randn(20000, 100)) + self.df3 = pd.DataFrame(np.random.randn(20000, 100)) + self.df4 = pd.DataFrame(np.random.randn(20000, 100)) + + if threads == 1: + expr.set_numexpr_threads(1) + + def time_add(self, engine, threads): + pd.eval("self.df + self.df2 + self.df3 + self.df4", engine=engine) + + def time_and(self, engine, threads): + pd.eval( + "(self.df > 0) & (self.df2 > 0) & (self.df3 > 0) & (self.df4 > 0)", + engine=engine, + ) + + def time_chained_cmp(self, engine, threads): + pd.eval("self.df < self.df2 < self.df3 < self.df4", engine=engine) + + def time_mult(self, engine, threads): + pd.eval("self.df * self.df2 * self.df3 * self.df4", engine=engine) + + def teardown(self, engine, threads): + expr.set_numexpr_threads() + + +class Query: + def setup(self): + N = 10**6 + halfway = (N // 2) - 1 + index = pd.date_range("20010101", periods=N, freq="min") + s = pd.Series(index) + self.ts = s.iloc[halfway] + self.df = pd.DataFrame({"a": np.random.randn(N), "dates": index}, index=index) + data = np.random.randn(N) + self.min_val = data.min() + self.max_val = data.max() + + def time_query_datetime_index(self): + self.df.query("index < @self.ts") + + def time_query_datetime_column(self): + self.df.query("dates < @self.ts") + + def time_query_with_boolean_selection(self): + self.df.query("(a >= @self.min_val) & (a <= @self.max_val)") + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/finalize.py b/asv_bench/benchmarks/finalize.py new file mode 100644 index 0000000000000..dc06f55cc6ca0 --- /dev/null +++ b/asv_bench/benchmarks/finalize.py @@ -0,0 +1,16 @@ +import pandas as pd + + +class Finalize: + param_names = ["series", "frame"] + params = [pd.Series, pd.DataFrame] + + def setup(self, param): + N = 1000 + obj = param(dtype=float) + for i in range(N): + obj.attrs[i] = i + self.obj = obj + + def time_finalize_micro(self, param): + self.obj.__finalize__(self.obj, method="__finalize__") diff --git a/asv_bench/benchmarks/frame_ctor.py b/asv_bench/benchmarks/frame_ctor.py new file mode 100644 index 0000000000000..f938f7eb0d951 --- /dev/null +++ b/asv_bench/benchmarks/frame_ctor.py @@ -0,0 +1,201 @@ +import numpy as np + +import pandas as pd +from pandas import ( + NA, + Categorical, + DataFrame, + Float64Dtype, + MultiIndex, + Series, + Timestamp, + date_range, +) + +try: + from pandas.tseries.offsets import ( + Hour, + Nano, + ) +except ImportError: + # For compatibility with older versions + from pandas.core.datetools import ( + Hour, + Nano, + ) + + +class FromDicts: + def setup(self): + N, K = 5000, 50 + self.index = pd.Index([f"i-{i}" for i in range(N)], dtype=object) + self.columns = pd.Index([f"i-{i}" for i in range(K)], dtype=object) + frame = DataFrame(np.random.randn(N, K), index=self.index, columns=self.columns) + self.data = frame.to_dict() + self.dict_list = frame.to_dict(orient="records") + self.data2 = {i: {j: float(j) for j in range(100)} for i in range(2000)} + + # arrays which we won't consolidate + self.dict_of_categoricals = {i: Categorical(np.arange(N)) for i in range(K)} + + def time_list_of_dict(self): + DataFrame(self.dict_list) + + def time_nested_dict(self): + DataFrame(self.data) + + def time_nested_dict_index(self): + DataFrame(self.data, index=self.index) + + def time_nested_dict_columns(self): + DataFrame(self.data, columns=self.columns) + + def time_nested_dict_index_columns(self): + DataFrame(self.data, index=self.index, columns=self.columns) + + def time_nested_dict_int64(self): + # nested dict, integer indexes, regression described in #621 + DataFrame(self.data2) + + def time_dict_of_categoricals(self): + # dict of arrays that we won't consolidate + DataFrame(self.dict_of_categoricals) + + +class FromSeries: + def setup(self): + mi = MultiIndex.from_product([range(100), range(100)]) + self.s = Series(np.random.randn(10000), index=mi) + + def time_mi_series(self): + DataFrame(self.s) + + +class FromDictwithTimestamp: + params = [Nano(1), Hour(1)] + param_names = ["offset"] + + def setup(self, offset): + N = 10**3 + idx = date_range(Timestamp("1/1/1900"), freq=offset, periods=N) + df = DataFrame(np.random.randn(N, 10), index=idx) + self.d = df.to_dict() + + def time_dict_with_timestamp_offsets(self, offset): + DataFrame(self.d) + + +class FromRecords: + params = [None, 1000] + param_names = ["nrows"] + + # Generators get exhausted on use, so run setup before every call + number = 1 + repeat = (3, 250, 10) + + def setup(self, nrows): + N = 100000 + self.gen = ((x, (x * 20), (x * 100)) for x in range(N)) + + def time_frame_from_records_generator(self, nrows): + # issue-6700 + self.df = DataFrame.from_records(self.gen, nrows=nrows) + + +class FromNDArray: + def setup(self): + N = 100000 + self.data = np.random.randn(N) + + def time_frame_from_ndarray(self): + self.df = DataFrame(self.data) + + +class FromLists: + goal_time = 0.2 + + def setup(self): + N = 1000 + M = 100 + self.data = [list(range(M)) for i in range(N)] + + def time_frame_from_lists(self): + self.df = DataFrame(self.data) + + +class FromRange: + goal_time = 0.2 + + def setup(self): + N = 1_000_000 + self.data = range(N) + + def time_frame_from_range(self): + self.df = DataFrame(self.data) + + +class FromScalar: + def setup(self): + self.nrows = 100_000 + + def time_frame_from_scalar_ea_float64(self): + DataFrame( + 1.0, + index=range(self.nrows), + columns=list("abc"), + dtype=Float64Dtype(), + ) + + def time_frame_from_scalar_ea_float64_na(self): + DataFrame( + NA, + index=range(self.nrows), + columns=list("abc"), + dtype=Float64Dtype(), + ) + + +class FromArrays: + goal_time = 0.2 + + def setup(self): + N_rows = 1000 + N_cols = 1000 + self.float_arrays = [np.random.randn(N_rows) for _ in range(N_cols)] + self.sparse_arrays = [ + pd.arrays.SparseArray(np.random.randint(0, 2, N_rows), dtype="float64") + for _ in range(N_cols) + ] + self.int_arrays = [ + pd.array(np.random.randint(1000, size=N_rows), dtype="Int64") + for _ in range(N_cols) + ] + self.index = pd.Index(range(N_rows)) + self.columns = pd.Index(range(N_cols)) + + def time_frame_from_arrays_float(self): + self.df = DataFrame._from_arrays( + self.float_arrays, + index=self.index, + columns=self.columns, + verify_integrity=False, + ) + + def time_frame_from_arrays_int(self): + self.df = DataFrame._from_arrays( + self.int_arrays, + index=self.index, + columns=self.columns, + verify_integrity=False, + ) + + def time_frame_from_arrays_sparse(self): + self.df = DataFrame._from_arrays( + self.sparse_arrays, + index=self.index, + columns=self.columns, + verify_integrity=False, + ) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/frame_methods.py b/asv_bench/benchmarks/frame_methods.py new file mode 100644 index 0000000000000..cd7851acae3f2 --- /dev/null +++ b/asv_bench/benchmarks/frame_methods.py @@ -0,0 +1,889 @@ +import string +import warnings + +import numpy as np + +from pandas import ( + DataFrame, + Index, + MultiIndex, + NaT, + Series, + date_range, + isnull, + period_range, + timedelta_range, +) + + +class AsType: + params = [ + [ + # from_dtype == to_dtype + ("Float64", "Float64"), + ("float64[pyarrow]", "float64[pyarrow]"), + # from non-EA to EA + ("float64", "Float64"), + ("float64", "float64[pyarrow]"), + # from EA to non-EA + ("Float64", "float64"), + ("float64[pyarrow]", "float64"), + # from EA to EA + ("Int64", "Float64"), + ("int64[pyarrow]", "float64[pyarrow]"), + ], + [False, True], + ] + param_names = ["from_to_dtypes", "copy"] + + def setup(self, from_to_dtypes, copy): + from_dtype = from_to_dtypes[0] + if from_dtype in ("float64", "Float64", "float64[pyarrow]"): + data = np.random.randn(100, 100) + elif from_dtype in ("int64", "Int64", "int64[pyarrow]"): + data = np.random.randint(0, 1000, (100, 100)) + else: + raise NotImplementedError + self.df = DataFrame(data, dtype=from_dtype) + + def time_astype(self, from_to_dtypes, copy): + self.df.astype(from_to_dtypes[1], copy=copy) + + +class Clip: + params = [ + ["float64", "Float64", "float64[pyarrow]"], + ] + param_names = ["dtype"] + + def setup(self, dtype): + data = np.random.randn(100_000, 10) + df = DataFrame(data, dtype=dtype) + self.df = df + + def time_clip(self, dtype): + self.df.clip(-1.0, 1.0) + + +class GetNumericData: + def setup(self): + self.df = DataFrame(np.random.randn(10000, 25)) + self.df["foo"] = "bar" + self.df["bar"] = "baz" + self.df = self.df._consolidate() + + def time_frame_get_numeric_data(self): + self.df._get_numeric_data() + + +class Reindex: + def setup(self): + N = 10**3 + self.df = DataFrame(np.random.randn(N * 10, N)) + self.idx = np.arange(4 * N, 7 * N) + self.idx_cols = np.random.randint(0, N, N) + self.df2 = DataFrame( + { + c: { + 0: np.random.randint(0, 2, N).astype(np.bool_), + 1: np.random.randint(0, N, N).astype(np.int16), + 2: np.random.randint(0, N, N).astype(np.int32), + 3: np.random.randint(0, N, N).astype(np.int64), + }[np.random.randint(0, 4)] + for c in range(N) + } + ) + + def time_reindex_axis0(self): + self.df.reindex(self.idx) + + def time_reindex_axis1(self): + self.df.reindex(columns=self.idx_cols) + + def time_reindex_axis1_missing(self): + self.df.reindex(columns=self.idx) + + def time_reindex_both_axes(self): + self.df.reindex(index=self.idx, columns=self.idx_cols) + + def time_reindex_upcast(self): + self.df2.reindex(np.random.permutation(range(1200))) + + +class Rename: + def setup(self): + N = 10**3 + self.df = DataFrame(np.random.randn(N * 10, N)) + self.idx = np.arange(4 * N, 7 * N) + self.dict_idx = {k: k for k in self.idx} + self.df2 = DataFrame( + { + c: { + 0: np.random.randint(0, 2, N).astype(np.bool_), + 1: np.random.randint(0, N, N).astype(np.int16), + 2: np.random.randint(0, N, N).astype(np.int32), + 3: np.random.randint(0, N, N).astype(np.int64), + }[np.random.randint(0, 4)] + for c in range(N) + } + ) + + def time_rename_single(self): + self.df.rename({0: 0}) + + def time_rename_axis0(self): + self.df.rename(self.dict_idx) + + def time_rename_axis1(self): + self.df.rename(columns=self.dict_idx) + + def time_rename_both_axes(self): + self.df.rename(index=self.dict_idx, columns=self.dict_idx) + + def time_dict_rename_both_axes(self): + self.df.rename(index=self.dict_idx, columns=self.dict_idx) + + +class Iteration: + # mem_itertuples_* benchmarks are slow + timeout = 120 + + def setup(self): + N = 1000 + self.df = DataFrame(np.random.randn(N * 10, N)) + self.df2 = DataFrame(np.random.randn(N * 50, 10)) + self.df3 = DataFrame( + np.random.randn(N, 5 * N), columns=["C" + str(c) for c in range(N * 5)] + ) + self.df4 = DataFrame(np.random.randn(N * 1000, 10)) + + def time_items(self): + # (monitor no-copying behaviour) + for name, col in self.df.items(): + pass + + def time_iteritems_indexing(self): + for col in self.df3: + self.df3[col] + + def time_itertuples_start(self): + self.df4.itertuples() + + def time_itertuples_read_first(self): + next(self.df4.itertuples()) + + def time_itertuples(self): + for row in self.df4.itertuples(): + pass + + def time_itertuples_to_list(self): + list(self.df4.itertuples()) + + def mem_itertuples_start(self): + return self.df4.itertuples() + + def peakmem_itertuples_start(self): + self.df4.itertuples() + + def mem_itertuples_read_first(self): + return next(self.df4.itertuples()) + + def peakmem_itertuples(self): + for row in self.df4.itertuples(): + pass + + def mem_itertuples_to_list(self): + return list(self.df4.itertuples()) + + def peakmem_itertuples_to_list(self): + list(self.df4.itertuples()) + + def time_itertuples_raw_start(self): + self.df4.itertuples(index=False, name=None) + + def time_itertuples_raw_read_first(self): + next(self.df4.itertuples(index=False, name=None)) + + def time_itertuples_raw_tuples(self): + for row in self.df4.itertuples(index=False, name=None): + pass + + def time_itertuples_raw_tuples_to_list(self): + list(self.df4.itertuples(index=False, name=None)) + + def mem_itertuples_raw_start(self): + return self.df4.itertuples(index=False, name=None) + + def peakmem_itertuples_raw_start(self): + self.df4.itertuples(index=False, name=None) + + def peakmem_itertuples_raw_read_first(self): + next(self.df4.itertuples(index=False, name=None)) + + def peakmem_itertuples_raw(self): + for row in self.df4.itertuples(index=False, name=None): + pass + + def mem_itertuples_raw_to_list(self): + return list(self.df4.itertuples(index=False, name=None)) + + def peakmem_itertuples_raw_to_list(self): + list(self.df4.itertuples(index=False, name=None)) + + def time_iterrows(self): + for row in self.df.iterrows(): + pass + + +class ToString: + def setup(self): + self.df = DataFrame(np.random.randn(100, 10)) + + def time_to_string_floats(self): + self.df.to_string() + + +class ToHTML: + def setup(self): + nrows = 500 + self.df2 = DataFrame(np.random.randn(nrows, 10)) + self.df2[0] = period_range("2000", periods=nrows) + self.df2[1] = range(nrows) + + def time_to_html_mixed(self): + self.df2.to_html() + + +class ToDict: + params = [["dict", "list", "series", "split", "records", "index"]] + param_names = ["orient"] + + def setup(self, orient): + data = np.random.randint(0, 1000, size=(10000, 4)) + self.int_df = DataFrame(data) + self.datetimelike_df = self.int_df.astype("timedelta64[ns]") + + def time_to_dict_ints(self, orient): + self.int_df.to_dict(orient=orient) + + def time_to_dict_datetimelike(self, orient): + self.datetimelike_df.to_dict(orient=orient) + + +class ToNumpy: + def setup(self): + N = 10000 + M = 10 + self.df_tall = DataFrame(np.random.randn(N, M)) + self.df_wide = DataFrame(np.random.randn(M, N)) + self.df_mixed_tall = self.df_tall.copy() + self.df_mixed_tall["foo"] = "bar" + self.df_mixed_tall[0] = period_range("2000", periods=N) + self.df_mixed_tall[1] = range(N) + self.df_mixed_wide = self.df_wide.copy() + self.df_mixed_wide["foo"] = "bar" + self.df_mixed_wide[0] = period_range("2000", periods=M) + self.df_mixed_wide[1] = range(M) + + def time_to_numpy_tall(self): + self.df_tall.to_numpy() + + def time_to_numpy_wide(self): + self.df_wide.to_numpy() + + def time_to_numpy_mixed_tall(self): + self.df_mixed_tall.to_numpy() + + def time_to_numpy_mixed_wide(self): + self.df_mixed_wide.to_numpy() + + def time_values_tall(self): + self.df_tall.values + + def time_values_wide(self): + self.df_wide.values + + def time_values_mixed_tall(self): + self.df_mixed_tall.values + + def time_values_mixed_wide(self): + self.df_mixed_wide.values + + +class ToRecords: + def setup(self): + N = 100_000 + data = np.random.randn(N, 2) + mi = MultiIndex.from_arrays( + [ + np.arange(N), + date_range("1970-01-01", periods=N, freq="ms"), + ] + ) + self.df = DataFrame(data) + self.df_mi = DataFrame(data, index=mi) + + def time_to_records(self): + self.df.to_records(index=True) + + def time_to_records_multiindex(self): + self.df_mi.to_records(index=True) + + +class Repr: + def setup(self): + nrows = 10000 + data = np.random.randn(nrows, 10) + arrays = np.tile(np.random.randn(3, nrows // 100), 100) + idx = MultiIndex.from_arrays(arrays) + self.df3 = DataFrame(data, index=idx) + self.df4 = DataFrame(data, index=np.random.randn(nrows)) + self.df_tall = DataFrame(np.random.randn(nrows, 10)) + self.df_wide = DataFrame(np.random.randn(10, nrows)) + + def time_html_repr_trunc_mi(self): + self.df3._repr_html_() + + def time_html_repr_trunc_si(self): + self.df4._repr_html_() + + def time_repr_tall(self): + repr(self.df_tall) + + def time_frame_repr_wide(self): + repr(self.df_wide) + + +class MaskBool: + def setup(self): + data = np.random.randn(1000, 500) + df = DataFrame(data) + df = df.where(df > 0) + self.bools = df > 0 + self.mask = isnull(df) + + def time_frame_mask_bools(self): + self.bools.mask(self.mask) + + def time_frame_mask_floats(self): + self.bools.astype(float).mask(self.mask) + + +class Isnull: + def setup(self): + N = 10**3 + self.df_no_null = DataFrame(np.random.randn(N, N)) + + sample = np.array([np.nan, 1.0]) + data = np.random.choice(sample, (N, N)) + self.df = DataFrame(data) + + sample = np.array(list(string.ascii_letters + string.whitespace)) + data = np.random.choice(sample, (N, N)) + self.df_strings = DataFrame(data) + + sample = np.array( + [ + NaT, + np.nan, + None, + np.datetime64("NaT"), + np.timedelta64("NaT"), + 0, + 1, + 2.0, + "", + "abcd", + ] + ) + data = np.random.choice(sample, (N, N)) + self.df_obj = DataFrame(data) + + def time_isnull_floats_no_null(self): + isnull(self.df_no_null) + + def time_isnull(self): + isnull(self.df) + + def time_isnull_strngs(self): + isnull(self.df_strings) + + def time_isnull_obj(self): + isnull(self.df_obj) + + +class Fillna: + params = ( + [True, False], + [ + "float64", + "float32", + "object", + "Int64", + "Float64", + "datetime64[ns]", + "datetime64[ns, tz]", + "timedelta64[ns]", + ], + ) + param_names = ["inplace", "dtype"] + + def setup(self, inplace, dtype): + N, M = 10000, 100 + if dtype in ("datetime64[ns]", "datetime64[ns, tz]", "timedelta64[ns]"): + data = { + "datetime64[ns]": date_range("2011-01-01", freq="h", periods=N), + "datetime64[ns, tz]": date_range( + "2011-01-01", freq="h", periods=N, tz="Asia/Tokyo" + ), + "timedelta64[ns]": timedelta_range(start="1 day", periods=N, freq="1D"), + } + self.df = DataFrame({f"col_{i}": data[dtype] for i in range(M)}) + self.df[::2] = None + else: + values = np.random.randn(N, M) + values[::2] = np.nan + if dtype == "Int64": + values = values.round() + self.df = DataFrame(values, dtype=dtype) + self.fill_values = self.df.iloc[self.df.first_valid_index()].to_dict() + + def time_fillna(self, inplace, dtype): + self.df.fillna(value=self.fill_values, inplace=inplace) + + def time_ffill(self, inplace, dtype): + self.df.ffill(inplace=inplace) + + def time_bfill(self, inplace, dtype): + self.df.bfill(inplace=inplace) + + +class Dropna: + params = (["all", "any"], [0, 1]) + param_names = ["how", "axis"] + + def setup(self, how, axis): + self.df = DataFrame(np.random.randn(10000, 1000)) + self.df.iloc[50:1000, 20:50] = np.nan + self.df.iloc[2000:3000] = np.nan + self.df.iloc[:, 60:70] = np.nan + self.df_mixed = self.df.copy() + self.df_mixed["foo"] = "bar" + + def time_dropna(self, how, axis): + self.df.dropna(how=how, axis=axis) + + def time_dropna_axis_mixed_dtypes(self, how, axis): + self.df_mixed.dropna(how=how, axis=axis) + + +class Isna: + params = ["float64", "Float64", "float64[pyarrow]"] + param_names = ["dtype"] + + def setup(self, dtype): + data = np.random.randn(10000, 1000) + # all-na columns + data[:, 600:800] = np.nan + # partial-na columns + data[800:1000, 4000:5000] = np.nan + self.df = DataFrame(data, dtype=dtype) + + def time_isna(self, dtype): + self.df.isna() + + +class Count: + params = [0, 1] + param_names = ["axis"] + + def setup(self, axis): + self.df = DataFrame(np.random.randn(10000, 1000)) + self.df.iloc[50:1000, 20:50] = np.nan + self.df.iloc[2000:3000] = np.nan + self.df.iloc[:, 60:70] = np.nan + self.df_mixed = self.df.copy() + self.df_mixed["foo"] = "bar" + + def time_count(self, axis): + self.df.count(axis=axis) + + def time_count_mixed_dtypes(self, axis): + self.df_mixed.count(axis=axis) + + +class Apply: + def setup(self): + self.df = DataFrame(np.random.randn(1000, 100)) + + self.s = Series(np.arange(1028.0)) + self.df2 = DataFrame(dict.fromkeys(range(1028), self.s)) + self.df3 = DataFrame(np.random.randn(1000, 3), columns=list("ABC")) + + def time_apply_user_func(self): + self.df2.apply(lambda x: np.corrcoef(x, self.s)[(0, 1)]) + + def time_apply_axis_1(self): + self.df.apply(lambda x: x + 1, axis=1) + + def time_apply_lambda_mean(self): + self.df.apply(lambda x: x.mean()) + + def time_apply_str_mean(self): + self.df.apply("mean") + + def time_apply_pass_thru(self): + self.df.apply(lambda x: x) + + def time_apply_ref_by_name(self): + self.df3.apply(lambda x: x["A"] + x["B"], axis=1) + + +class Dtypes: + def setup(self): + self.df = DataFrame(np.random.randn(1000, 1000)) + + def time_frame_dtypes(self): + self.df.dtypes + + +class Equals: + def setup(self): + N = 10**3 + self.float_df = DataFrame(np.random.randn(N, N)) + self.float_df_nan = self.float_df.copy() + self.float_df_nan.iloc[-1, -1] = np.nan + + self.object_df = DataFrame("foo", index=range(N), columns=range(N)) + self.object_df_nan = self.object_df.copy() + self.object_df_nan.iloc[-1, -1] = np.nan + + self.nonunique_cols = self.object_df.copy() + self.nonunique_cols.columns = ["A"] * len(self.nonunique_cols.columns) + self.nonunique_cols_nan = self.nonunique_cols.copy() + self.nonunique_cols_nan.iloc[-1, -1] = np.nan + + def time_frame_float_equal(self): + self.float_df.equals(self.float_df) + + def time_frame_float_unequal(self): + self.float_df.equals(self.float_df_nan) + + def time_frame_nonunique_equal(self): + self.nonunique_cols.equals(self.nonunique_cols) + + def time_frame_nonunique_unequal(self): + self.nonunique_cols.equals(self.nonunique_cols_nan) + + def time_frame_object_equal(self): + self.object_df.equals(self.object_df) + + def time_frame_object_unequal(self): + self.object_df.equals(self.object_df_nan) + + +class Interpolate: + def setup(self): + N = 10000 + # this is the worst case, where every column has NaNs. + arr = np.random.randn(N, 100) + arr[::2] = np.nan + + self.df = DataFrame(arr) + + self.df2 = DataFrame( + { + "A": np.arange(0, N), + "B": np.random.randint(0, 100, N), + "C": np.random.randn(N), + "D": np.random.randn(N), + } + ) + self.df2.loc[1::5, "A"] = np.nan + self.df2.loc[1::5, "C"] = np.nan + + def time_interpolate(self): + self.df.interpolate() + + def time_interpolate_some_good(self): + self.df2.interpolate() + + +class Shift: + # frame shift speedup issue-5609 + params = [0, 1] + param_names = ["axis"] + + def setup(self, axis): + self.df = DataFrame(np.random.rand(10000, 500)) + + def time_shift(self, axis): + self.df.shift(1, axis=axis) + + +class Nunique: + def setup(self): + self.df = DataFrame(np.random.randn(10000, 1000)) + + def time_frame_nunique(self): + self.df.nunique() + + +class SeriesNuniqueWithNan: + def setup(self): + values = 100 * [np.nan] + list(range(100)) + self.ser = Series(np.tile(values, 10000), dtype=float) + + def time_series_nunique_nan(self): + self.ser.nunique() + + +class Duplicated: + def setup(self): + n = 1 << 20 + t = date_range("2015-01-01", freq="s", periods=(n // 64)) + xs = np.random.randn(n // 64).round(2) + self.df = DataFrame( + { + "a": np.random.randint(-1 << 8, 1 << 8, n), + "b": np.random.choice(t, n), + "c": np.random.choice(xs, n), + } + ) + self.df2 = DataFrame(np.random.randn(1000, 100).astype(str)).T + + def time_frame_duplicated(self): + self.df.duplicated() + + def time_frame_duplicated_wide(self): + self.df2.duplicated() + + def time_frame_duplicated_subset(self): + self.df.duplicated(subset=["a"]) + + +class XS: + params = [0, 1] + param_names = ["axis"] + + def setup(self, axis): + self.N = 10**4 + self.df = DataFrame(np.random.randn(self.N, self.N)) + + def time_frame_xs(self, axis): + self.df.xs(self.N / 2, axis=axis) + + +class SortValues: + params = [True, False] + param_names = ["ascending"] + + def setup(self, ascending): + self.df = DataFrame(np.random.randn(1000000, 2), columns=list("AB")) + + def time_frame_sort_values(self, ascending): + self.df.sort_values(by="A", ascending=ascending) + + +class SortMultiKey: + params = [True, False] + param_names = ["monotonic"] + + def setup(self, monotonic): + N = 10000 + K = 10 + df = DataFrame( + { + "key1": Index([f"i-{i}" for i in range(N)], dtype=object).values.repeat( + K + ), + "key2": Index([f"i-{i}" for i in range(N)], dtype=object).values.repeat( + K + ), + "value": np.random.randn(N * K), + } + ) + if monotonic: + df = df.sort_values(["key1", "key2"]) + self.df_by_columns = df + self.df_by_index = df.set_index(["key1", "key2"]) + + def time_sort_values(self, monotonic): + self.df_by_columns.sort_values(by=["key1", "key2"]) + + def time_sort_index(self, monotonic): + self.df_by_index.sort_index() + + +class Quantile: + params = [0, 1] + param_names = ["axis"] + + def setup(self, axis): + self.df = DataFrame(np.random.randn(1000, 3), columns=list("ABC")) + + def time_frame_quantile(self, axis): + self.df.quantile([0.1, 0.5], axis=axis) + + +class Rank: + param_names = ["dtype"] + params = [ + ["int", "uint", "float", "object"], + ] + + def setup(self, dtype): + self.df = DataFrame( + np.random.randn(10000, 10).astype(dtype), columns=range(10), dtype=dtype + ) + + def time_rank(self, dtype): + self.df.rank() + + +class GetDtypeCounts: + # 2807 + def setup(self): + self.df = DataFrame(np.random.randn(10, 10000)) + + def time_frame_get_dtype_counts(self): + with warnings.catch_warnings(record=True): + self.df.dtypes.value_counts() + + def time_info(self): + self.df.info() + + +class NSort: + params = ["first", "last", "all"] + param_names = ["keep"] + + def setup(self, keep): + self.df = DataFrame(np.random.randn(100000, 3), columns=list("ABC")) + + def time_nlargest_one_column(self, keep): + self.df.nlargest(100, "A", keep=keep) + + def time_nlargest_two_columns(self, keep): + self.df.nlargest(100, ["A", "B"], keep=keep) + + def time_nsmallest_one_column(self, keep): + self.df.nsmallest(100, "A", keep=keep) + + def time_nsmallest_two_columns(self, keep): + self.df.nsmallest(100, ["A", "B"], keep=keep) + + +class Describe: + def setup(self): + self.df = DataFrame( + { + "a": np.random.randint(0, 100, 10**6), + "b": np.random.randint(0, 100, 10**6), + "c": np.random.randint(0, 100, 10**6), + } + ) + + def time_series_describe(self): + self.df["a"].describe() + + def time_dataframe_describe(self): + self.df.describe() + + +class MemoryUsage: + def setup(self): + self.df = DataFrame(np.random.randn(100000, 2), columns=list("AB")) + self.df2 = self.df.copy() + self.df2["A"] = self.df2["A"].astype("object") + + def time_memory_usage(self): + self.df.memory_usage(deep=True) + + def time_memory_usage_object_dtype(self): + self.df2.memory_usage(deep=True) + + +class Round: + def setup(self): + self.df = DataFrame(np.random.randn(10000, 10)) + self.df_t = self.df.transpose(copy=True) + + def time_round(self): + self.df.round() + + def time_round_transposed(self): + self.df_t.round() + + def peakmem_round(self): + self.df.round() + + def peakmem_round_transposed(self): + self.df_t.round() + + +class Where: + params = ( + [True, False], + ["float64", "Float64", "float64[pyarrow]"], + ) + param_names = ["dtype"] + + def setup(self, inplace, dtype): + self.df = DataFrame(np.random.randn(100_000, 10), dtype=dtype) + self.mask = self.df < 0 + + def time_where(self, inplace, dtype): + self.df.where(self.mask, other=0.0, inplace=inplace) + + +class FindValidIndex: + param_names = ["dtype"] + params = [ + ["float", "Float64", "float64[pyarrow]"], + ] + + def setup(self, dtype): + df = DataFrame( + np.random.randn(100000, 2), + columns=list("AB"), + dtype=dtype, + ) + df.iloc[:100, 0] = None + df.iloc[:200, 1] = None + df.iloc[-100:, 0] = None + df.iloc[-200:, 1] = None + self.df = df + + def time_first_valid_index(self, dtype): + self.df.first_valid_index() + + def time_last_valid_index(self, dtype): + self.df.last_valid_index() + + +class Update: + def setup(self): + rng = np.random.default_rng() + self.df = DataFrame(rng.uniform(size=(1_000_000, 10))) + + idx = rng.choice(range(1_000_000), size=1_000_000, replace=False) + self.df_random = DataFrame(self.df, index=idx) + + idx = rng.choice(range(1_000_000), size=100_000, replace=False) + cols = rng.choice(range(10), size=2, replace=False) + self.df_sample = DataFrame( + rng.uniform(size=(100_000, 2)), index=idx, columns=cols + ) + + def time_to_update_big_frame_small_arg(self): + self.df.update(self.df_sample) + + def time_to_update_random_indices(self): + self.df_random.update(self.df_sample) + + def time_to_update_small_frame_big_arg(self): + self.df_sample.update(self.df) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/gil.py b/asv_bench/benchmarks/gil.py new file mode 100644 index 0000000000000..a2f1db9ef6b87 --- /dev/null +++ b/asv_bench/benchmarks/gil.py @@ -0,0 +1,327 @@ +from functools import wraps +import threading + +import numpy as np + +from pandas import ( + DataFrame, + Index, + Series, + date_range, + factorize, + read_csv, +) +from pandas.core.algorithms import take_nd + +try: + from pandas import ( + rolling_kurt, + rolling_max, + rolling_mean, + rolling_median, + rolling_min, + rolling_skew, + rolling_std, + rolling_var, + ) + + have_rolling_methods = True +except ImportError: + have_rolling_methods = False +try: + from pandas._libs import algos +except ImportError: + from pandas import algos + +from .pandas_vb_common import BaseIO # isort:skip + + +def run_parallel(num_threads=2, kwargs_list=None): + """ + Decorator to run the same function multiple times in parallel. + + Parameters + ---------- + num_threads : int, optional + The number of times the function is run in parallel. + kwargs_list : list of dicts, optional + The list of kwargs to update original + function kwargs on different threads. + + Notes + ----- + This decorator does not pass the return value of the decorated function. + + Original from scikit-image: + + https://github.com/scikit-image/scikit-image/pull/1519 + + """ + assert num_threads > 0 + has_kwargs_list = kwargs_list is not None + if has_kwargs_list: + assert len(kwargs_list) == num_threads + + def wrapper(func): + @wraps(func) + def inner(*args, **kwargs): + if has_kwargs_list: + update_kwargs = lambda i: dict(kwargs, **kwargs_list[i]) + else: + update_kwargs = lambda i: kwargs + threads = [] + for i in range(num_threads): + updated_kwargs = update_kwargs(i) + thread = threading.Thread(target=func, args=args, kwargs=updated_kwargs) + threads.append(thread) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + return inner + + return wrapper + + +class ParallelGroupbyMethods: + params = ([2, 4, 8], ["count", "last", "max", "mean", "min", "prod", "sum", "var"]) + param_names = ["threads", "method"] + + def setup(self, threads, method): + N = 10**6 + ngroups = 10**3 + df = DataFrame( + {"key": np.random.randint(0, ngroups, size=N), "data": np.random.randn(N)} + ) + + @run_parallel(num_threads=threads) + def parallel(): + getattr(df.groupby("key")["data"], method)() + + self.parallel = parallel + + def loop(): + getattr(df.groupby("key")["data"], method)() + + self.loop = loop + + def time_parallel(self, threads, method): + self.parallel() + + def time_loop(self, threads, method): + for i in range(threads): + self.loop() + + +class ParallelGroups: + params = [2, 4, 8] + param_names = ["threads"] + + def setup(self, threads): + size = 2**22 + ngroups = 10**3 + data = Series(np.random.randint(0, ngroups, size=size)) + + @run_parallel(num_threads=threads) + def get_groups(): + data.groupby(data).groups + + self.get_groups = get_groups + + def time_get_groups(self, threads): + self.get_groups() + + +class ParallelTake1D: + params = ["int64", "float64"] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10**6 + df = DataFrame({"col": np.arange(N, dtype=dtype)}) + indexer = np.arange(100, len(df) - 100) + + @run_parallel(num_threads=2) + def parallel_take1d(): + take_nd(df["col"].values, indexer) + + self.parallel_take1d = parallel_take1d + + def time_take1d(self, dtype): + self.parallel_take1d() + + +class ParallelKth: + # This depends exclusively on code in _libs/, could go in libs.py + + number = 1 + repeat = 5 + + def setup(self): + N = 10**7 + k = 5 * 10**5 + kwargs_list = [{"arr": np.random.randn(N)}, {"arr": np.random.randn(N)}] + + @run_parallel(num_threads=2, kwargs_list=kwargs_list) + def parallel_kth_smallest(arr): + algos.kth_smallest(arr, k) + + self.parallel_kth_smallest = parallel_kth_smallest + + def time_kth_smallest(self): + self.parallel_kth_smallest() + + +class ParallelDatetimeFields: + def setup(self): + N = 10**6 + self.dti = date_range("1900-01-01", periods=N, freq="min") + self.period = self.dti.to_period("D") + + def time_datetime_field_year(self): + @run_parallel(num_threads=2) + def run(dti): + dti.year + + run(self.dti) + + def time_datetime_field_day(self): + @run_parallel(num_threads=2) + def run(dti): + dti.day + + run(self.dti) + + def time_datetime_field_daysinmonth(self): + @run_parallel(num_threads=2) + def run(dti): + dti.days_in_month + + run(self.dti) + + def time_datetime_field_normalize(self): + @run_parallel(num_threads=2) + def run(dti): + dti.normalize() + + run(self.dti) + + def time_datetime_to_period(self): + @run_parallel(num_threads=2) + def run(dti): + dti.to_period("s") + + run(self.dti) + + def time_period_to_datetime(self): + @run_parallel(num_threads=2) + def run(period): + period.to_timestamp() + + run(self.period) + + +class ParallelRolling: + params = ["median", "mean", "min", "max", "var", "skew", "kurt", "std"] + param_names = ["method"] + + def setup(self, method): + win = 100 + arr = np.random.rand(100000) + if hasattr(DataFrame, "rolling"): + df = DataFrame(arr).rolling(win) + + @run_parallel(num_threads=2) + def parallel_rolling(): + getattr(df, method)() + + self.parallel_rolling = parallel_rolling + elif have_rolling_methods: + rolling = { + "median": rolling_median, + "mean": rolling_mean, + "min": rolling_min, + "max": rolling_max, + "var": rolling_var, + "skew": rolling_skew, + "kurt": rolling_kurt, + "std": rolling_std, + } + + @run_parallel(num_threads=2) + def parallel_rolling(): + rolling[method](arr, win) + + self.parallel_rolling = parallel_rolling + else: + raise NotImplementedError + + def time_rolling(self, method): + self.parallel_rolling() + + +class ParallelReadCSV(BaseIO): + number = 1 + repeat = 5 + params = ["float", "object", "datetime"] + param_names = ["dtype"] + + def setup(self, dtype): + rows = 10000 + cols = 50 + if dtype == "float": + df = DataFrame(np.random.randn(rows, cols)) + elif dtype == "datetime": + df = DataFrame( + np.random.randn(rows, cols), index=date_range("1/1/2000", periods=rows) + ) + elif dtype == "object": + df = DataFrame( + "foo", index=range(rows), columns=["object%03d" for _ in range(5)] + ) + else: + raise NotImplementedError + + self.fname = f"__test_{dtype}__.csv" + df.to_csv(self.fname) + + @run_parallel(num_threads=2) + def parallel_read_csv(): + read_csv(self.fname) + + self.parallel_read_csv = parallel_read_csv + + def time_read_csv(self, dtype): + self.parallel_read_csv() + + +class ParallelFactorize: + number = 1 + repeat = 5 + params = [2, 4, 8] + param_names = ["threads"] + + def setup(self, threads): + strings = Index([f"i-{i}" for i in range(100000)], dtype=object) + + @run_parallel(num_threads=threads) + def parallel(): + factorize(strings) + + self.parallel = parallel + + def loop(): + factorize(strings) + + self.loop = loop + + def time_parallel(self, threads): + self.parallel() + + def time_loop(self, threads): + for i in range(threads): + self.loop() + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/groupby.py b/asv_bench/benchmarks/groupby.py new file mode 100644 index 0000000000000..19c556dfe9d1f --- /dev/null +++ b/asv_bench/benchmarks/groupby.py @@ -0,0 +1,1151 @@ +from functools import partial +from itertools import product +from string import ascii_letters + +import numpy as np + +from pandas import ( + NA, + Categorical, + DataFrame, + Index, + MultiIndex, + Series, + Timestamp, + date_range, + period_range, + to_timedelta, +) + +method_blocklist = { + "object": { + "diff", + "median", + "prod", + "sem", + "cumsum", + "sum", + "cummin", + "mean", + "max", + "skew", + "cumprod", + "cummax", + "pct_change", + "min", + "var", + "describe", + "std", + "quantile", + }, + "datetime": { + "median", + "prod", + "sem", + "cumsum", + "sum", + "mean", + "skew", + "cumprod", + "cummax", + "pct_change", + "var", + "describe", + "std", + }, +} + +# These aggregations don't have a kernel implemented for them yet +_numba_unsupported_methods = [ + "all", + "any", + "bfill", + "count", + "cumcount", + "cummax", + "cummin", + "cumprod", + "cumsum", + "describe", + "diff", + "ffill", + "first", + "head", + "idxmax", + "idxmin", + "last", + "median", + "nunique", + "pct_change", + "prod", + "quantile", + "rank", + "sem", + "shift", + "size", + "skew", + "tail", + "unique", + "value_counts", +] + + +class ApplyDictReturn: + def setup(self): + self.labels = np.arange(1000).repeat(10) + self.data = Series(np.random.randn(len(self.labels))) + + def time_groupby_apply_dict_return(self): + self.data.groupby(self.labels).apply( + lambda x: {"first": x.values[0], "last": x.values[-1]} + ) + + +class Apply: + param_names = ["factor"] + params = [4, 5] + + def setup(self, factor): + N = 10**factor + # two cases: + # - small groups: small data (N**4) + many labels (2000) -> average group + # size of 5 (-> larger overhead of slicing method) + # - larger groups: larger data (N**5) + fewer labels (20) -> average group + # size of 5000 + labels = np.random.randint(0, 2000 if factor == 4 else 20, size=N) + labels2 = np.random.randint(0, 3, size=N) + df = DataFrame( + { + "key": labels, + "key2": labels2, + "value1": np.random.randn(N), + "value2": ["foo", "bar", "baz", "qux"] * (N // 4), + } + ) + self.df = df + + def time_scalar_function_multi_col(self, factor): + self.df.groupby(["key", "key2"]).apply(lambda x: 1) + + def time_scalar_function_single_col(self, factor): + self.df.groupby("key").apply(lambda x: 1) + + @staticmethod + def df_copy_function(g): + # ensure that the group name is available (see GH #15062) + g.name + return g.copy() + + def time_copy_function_multi_col(self, factor): + self.df.groupby(["key", "key2"]).apply(self.df_copy_function) + + def time_copy_overhead_single_col(self, factor): + self.df.groupby("key").apply(self.df_copy_function) + + +class ApplyNonUniqueUnsortedIndex: + def setup(self): + # GH 46527 + # unsorted and non-unique index + idx = np.arange(100)[::-1] + idx = Index(np.repeat(idx, 200), name="key") + self.df = DataFrame(np.random.randn(len(idx), 10), index=idx) + + def time_groupby_apply_non_unique_unsorted_index(self): + self.df.groupby("key", group_keys=False).apply(lambda x: x) + + +class Groups: + param_names = ["key"] + params = ["int64_small", "int64_large", "object_small", "object_large"] + + def setup_cache(self): + size = 10**6 + data = { + "int64_small": Series(np.random.randint(0, 100, size=size)), + "int64_large": Series(np.random.randint(0, 10000, size=size)), + "object_small": Series( + Index([f"i-{i}" for i in range(100)], dtype=object).take( + np.random.randint(0, 100, size=size) + ) + ), + "object_large": Series( + Index([f"i-{i}" for i in range(10000)], dtype=object).take( + np.random.randint(0, 10000, size=size) + ) + ), + } + return data + + def setup(self, data, key): + self.ser = data[key] + + def time_series_groups(self, data, key): + self.ser.groupby(self.ser).groups + + def time_series_indices(self, data, key): + self.ser.groupby(self.ser).indices + + +class GroupManyLabels: + params = [1, 1000] + param_names = ["ncols"] + + def setup(self, ncols): + N = 1000 + data = np.random.randn(N, ncols) + self.labels = np.random.randint(0, 100, size=N) + self.df = DataFrame(data) + + def time_sum(self, ncols): + self.df.groupby(self.labels).sum() + + +class Nth: + param_names = ["dtype"] + params = ["float32", "float64", "datetime", "object"] + + def setup(self, dtype): + N = 10**5 + # with datetimes (GH7555) + if dtype == "datetime": + values = date_range("1/1/2011", periods=N, freq="s") + elif dtype == "object": + values = ["foo"] * N + else: + values = np.arange(N).astype(dtype) + + key = np.arange(N) + self.df = DataFrame({"key": key, "values": values}) + self.df.iloc[1, 1] = np.nan # insert missing data + + def time_frame_nth_any(self, dtype): + self.df.groupby("key").nth(0, dropna="any") + + def time_groupby_nth_all(self, dtype): + self.df.groupby("key").nth(0, dropna="all") + + def time_frame_nth(self, dtype): + self.df.groupby("key").nth(0) + + def time_series_nth_any(self, dtype): + self.df["values"].groupby(self.df["key"]).nth(0, dropna="any") + + def time_series_nth_all(self, dtype): + self.df["values"].groupby(self.df["key"]).nth(0, dropna="all") + + def time_series_nth(self, dtype): + self.df["values"].groupby(self.df["key"]).nth(0) + + +class DateAttributes: + def setup(self): + rng = date_range("1/1/2000", "12/31/2005", freq="h") + self.year, self.month, self.day = rng.year, rng.month, rng.day + self.ts = Series(np.random.randn(len(rng)), index=rng) + + def time_len_groupby_object(self): + len(self.ts.groupby([self.year, self.month, self.day])) + + +class Int64: + def setup(self): + arr = np.random.randint(-1 << 12, 1 << 12, (1 << 17, 5)) + i = np.random.choice(len(arr), len(arr) * 5) + arr = np.vstack((arr, arr[i])) + i = np.random.permutation(len(arr)) + arr = arr[i] + self.cols = list("abcde") + self.df = DataFrame(arr, columns=self.cols) + self.df["jim"], self.df["joe"] = np.random.randn(2, len(self.df)) * 10 + + def time_overflow(self): + self.df.groupby(self.cols).max() + + +class CountMultiDtype: + def setup_cache(self): + n = 10000 + offsets = np.random.randint(n, size=n).astype("timedelta64[ns]") + dates = np.datetime64("now") + offsets + dates[np.random.rand(n) > 0.5] = np.datetime64("nat") + offsets[np.random.rand(n) > 0.5] = np.timedelta64("nat") + value2 = np.random.randn(n) + value2[np.random.rand(n) > 0.5] = np.nan + obj = np.random.choice(list("ab"), size=n).astype(object) + obj[np.random.randn(n) > 0.5] = np.nan + df = DataFrame( + { + "key1": np.random.randint(0, 500, size=n), + "key2": np.random.randint(0, 100, size=n), + "dates": dates, + "value2": value2, + "value3": np.random.randn(n), + "ints": np.random.randint(0, 1000, size=n), + "obj": obj, + "offsets": offsets, + } + ) + return df + + def time_multi_count(self, df): + df.groupby(["key1", "key2"]).count() + + +class CountMultiInt: + def setup_cache(self): + n = 10000 + df = DataFrame( + { + "key1": np.random.randint(0, 500, size=n), + "key2": np.random.randint(0, 100, size=n), + "ints": np.random.randint(0, 1000, size=n), + "ints2": np.random.randint(0, 1000, size=n), + } + ) + return df + + def time_multi_int_count(self, df): + df.groupby(["key1", "key2"]).count() + + def time_multi_int_nunique(self, df): + df.groupby(["key1", "key2"]).nunique() + + +class AggFunctions: + def setup_cache(self): + N = 10**5 + fac1 = np.array(["A", "B", "C"], dtype="O") + fac2 = np.array(["one", "two"], dtype="O") + df = DataFrame( + { + "key1": fac1.take(np.random.randint(0, 3, size=N)), + "key2": fac2.take(np.random.randint(0, 2, size=N)), + "value1": np.random.randn(N), + "value2": np.random.randn(N), + "value3": np.random.randn(N), + } + ) + return df + + def time_different_str_functions(self, df): + df.groupby(["key1", "key2"]).agg( + {"value1": "mean", "value2": "var", "value3": "sum"} + ) + + def time_different_str_functions_multicol(self, df): + df.groupby(["key1", "key2"]).agg(["sum", "min", "max"]) + + def time_different_str_functions_singlecol(self, df): + df.groupby("key1").agg({"value1": "mean", "value2": "var", "value3": "sum"}) + + +class GroupStrings: + def setup(self): + n = 2 * 10**5 + alpha = list(map("".join, product(ascii_letters, repeat=4))) + data = np.random.choice(alpha, (n // 5, 4), replace=False) + data = np.repeat(data, 5, axis=0) + self.df = DataFrame(data, columns=list("abcd")) + self.df["joe"] = (np.random.randn(len(self.df)) * 10).round(3) + self.df = self.df.sample(frac=1).reset_index(drop=True) + + def time_multi_columns(self): + self.df.groupby(list("abcd")).max() + + +class MultiColumn: + def setup_cache(self): + N = 10**5 + key1 = np.tile(np.arange(100, dtype=object), 1000) + key2 = key1.copy() + np.random.shuffle(key1) + np.random.shuffle(key2) + df = DataFrame( + { + "key1": key1, + "key2": key2, + "data1": np.random.randn(N), + "data2": np.random.randn(N), + } + ) + return df + + def time_lambda_sum(self, df): + df.groupby(["key1", "key2"]).agg(lambda x: x.values.sum()) + + def time_cython_sum(self, df): + df.groupby(["key1", "key2"]).sum() + + def time_col_select_lambda_sum(self, df): + df.groupby(["key1", "key2"])["data1"].agg(lambda x: x.values.sum()) + + def time_col_select_str_sum(self, df): + df.groupby(["key1", "key2"])["data1"].agg("sum") + + +class Size: + def setup(self): + n = 10**5 + offsets = np.random.randint(n, size=n).astype("timedelta64[ns]") + dates = np.datetime64("now") + offsets + self.df = DataFrame( + { + "key1": np.random.randint(0, 500, size=n), + "key2": np.random.randint(0, 100, size=n), + "value1": np.random.randn(n), + "value2": np.random.randn(n), + "value3": np.random.randn(n), + "dates": dates, + } + ) + self.draws = Series(np.random.randn(n)) + labels = Series(["foo", "bar", "baz", "qux"] * (n // 4)) + self.cats = labels.astype("category") + + def time_multi_size(self): + self.df.groupby(["key1", "key2"]).size() + + def time_category_size(self): + self.draws.groupby(self.cats, observed=True).size() + + +class Shift: + def setup(self): + N = 18 + self.df = DataFrame({"g": ["a", "b"] * 9, "v": list(range(N))}) + + def time_defaults(self): + self.df.groupby("g").shift() + + def time_fill_value(self): + self.df.groupby("g").shift(fill_value=99) + + +class Fillna: + def setup(self): + N = 100 + self.df = DataFrame( + {"group": [1] * N + [2] * N, "value": [np.nan, 1.0] * N} + ).set_index("group") + + def time_df_ffill(self): + self.df.groupby("group").ffill() + + def time_df_bfill(self): + self.df.groupby("group").bfill() + + def time_srs_ffill(self): + self.df.groupby("group")["value"].ffill() + + def time_srs_bfill(self): + self.df.groupby("group")["value"].bfill() + + +class GroupByMethods: + param_names = ["dtype", "method", "application", "ncols"] + params = [ + ["int", "int16", "float", "object", "datetime", "uint"], + [ + "all", + "any", + "bfill", + "count", + "cumcount", + "cummax", + "cummin", + "cumprod", + "cumsum", + "describe", + "diff", + "ffill", + "first", + "head", + "last", + "max", + "min", + "median", + "mean", + "nunique", + "pct_change", + "prod", + "quantile", + "rank", + "sem", + "shift", + "size", + "skew", + "std", + "sum", + "tail", + "unique", + "value_counts", + "var", + ], + ["direct", "transformation"], + [1, 5], + ["cython", "numba"], + ] + + def setup(self, dtype, method, application, ncols, engine): + if method in method_blocklist.get(dtype, {}): + raise NotImplementedError # skip benchmark + + if ncols != 1 and method in ["value_counts", "unique"]: + # DataFrameGroupBy doesn't have these methods + raise NotImplementedError + + if application == "transformation" and method in [ + "describe", + "head", + "tail", + "unique", + "value_counts", + "size", + ]: + # DataFrameGroupBy doesn't have these methods + raise NotImplementedError + + # Numba currently doesn't support + # multiple transform functions or strs for transform, + # grouping on multiple columns + # and we lack kernels for a bunch of methods + if ( + (engine == "numba" and method in _numba_unsupported_methods) + or ncols > 1 + or application == "transformation" + or dtype == "datetime" + ): + raise NotImplementedError + + if method == "describe": + ngroups = 20 + elif method == "skew": + ngroups = 100 + else: + ngroups = 1000 + size = ngroups * 2 + rng = np.arange(ngroups).reshape(-1, 1) + rng = np.broadcast_to(rng, (len(rng), ncols)) + taker = np.random.randint(0, ngroups, size=size) + values = rng.take(taker, axis=0) + if dtype == "int": + key = np.random.randint(0, size, size=size) + elif dtype in ("int16", "uint"): + key = np.random.randint(0, size, size=size, dtype=dtype) + elif dtype == "float": + key = np.concatenate( + [np.random.random(ngroups) * 0.1, np.random.random(ngroups) * 10.0] + ) + elif dtype == "object": + key = ["foo"] * size + elif dtype == "datetime": + key = date_range("1/1/2011", periods=size, freq="s") + + cols = [f"values{n}" for n in range(ncols)] + df = DataFrame(values, columns=cols) + df["key"] = key + + if len(cols) == 1: + cols = cols[0] + + # Not everything supports the engine keyword yet + kwargs = {} + if engine == "numba": + kwargs["engine"] = engine + + if application == "transformation": + self.as_group_method = lambda: df.groupby("key")[cols].transform( + method, **kwargs + ) + self.as_field_method = lambda: df.groupby(cols)["key"].transform( + method, **kwargs + ) + else: + self.as_group_method = partial( + getattr(df.groupby("key")[cols], method), **kwargs + ) + self.as_field_method = partial( + getattr(df.groupby(cols)["key"], method), **kwargs + ) + + def time_dtype_as_group(self, dtype, method, application, ncols, engine): + self.as_group_method() + + def time_dtype_as_field(self, dtype, method, application, ncols, engine): + self.as_field_method() + + +class GroupByCythonAgg: + """ + Benchmarks specifically targeting our cython aggregation algorithms + (using a big enough dataframe with simple key, so a large part of the + time is actually spent in the grouped aggregation). + """ + + param_names = ["dtype", "method"] + params = [ + ["float64"], + [ + "sum", + "prod", + "min", + "max", + "idxmin", + "idxmax", + "mean", + "median", + "var", + "first", + "last", + "any", + "all", + ], + ] + + def setup(self, dtype, method): + N = 1_000_000 + df = DataFrame(np.random.randn(N, 10), columns=list("abcdefghij")) + df["key"] = np.random.randint(0, 100, size=N) + self.df = df + + def time_frame_agg(self, dtype, method): + self.df.groupby("key").agg(method) + + +class GroupByNumbaAgg(GroupByCythonAgg): + """ + Benchmarks specifically targeting our numba aggregation algorithms + (using a big enough dataframe with simple key, so a large part of the + time is actually spent in the grouped aggregation). + """ + + def setup(self, dtype, method): + if method in _numba_unsupported_methods: + raise NotImplementedError + super().setup(dtype, method) + + def time_frame_agg(self, dtype, method): + self.df.groupby("key").agg(method, engine="numba") + + +class GroupByCythonAggEaDtypes: + """ + Benchmarks specifically targeting our cython aggregation algorithms + (using a big enough dataframe with simple key, so a large part of the + time is actually spent in the grouped aggregation). + """ + + param_names = ["dtype", "method"] + params = [ + ["Float64", "Int64", "Int32"], + [ + "sum", + "prod", + "min", + "max", + "mean", + "median", + "var", + "first", + "last", + "any", + "all", + ], + ] + + def setup(self, dtype, method): + N = 1_000_000 + df = DataFrame( + np.random.randint(0, high=100, size=(N, 10)), + columns=list("abcdefghij"), + dtype=dtype, + ) + df.loc[list(range(1, N, 5)), list("abcdefghij")] = NA + df["key"] = np.random.randint(0, 100, size=N) + self.df = df + + def time_frame_agg(self, dtype, method): + self.df.groupby("key").agg(method) + + +class Cumulative: + param_names = ["dtype", "method", "with_nans"] + params = [ + ["float64", "int64", "Float64", "Int64"], + ["cummin", "cummax", "cumsum"], + [True, False], + ] + + def setup(self, dtype, method, with_nans): + if with_nans and dtype == "int64": + raise NotImplementedError("Construction of df would raise") + + N = 500_000 + keys = np.random.randint(0, 100, size=N) + vals = np.random.randint(-10, 10, (N, 5)) + + if with_nans: + null_vals = vals.astype(float, copy=True) + null_vals[::2, :] = np.nan + null_vals[::3, :] = np.nan + df = DataFrame(null_vals, columns=list("abcde"), dtype=dtype) + df["key"] = keys + self.df = df + else: + df = DataFrame(vals, columns=list("abcde")).astype(dtype, copy=False) + df["key"] = keys + self.df = df + + def time_frame_transform(self, dtype, method, with_nans): + self.df.groupby("key").transform(method) + + +class RankWithTies: + # GH 21237 + param_names = ["dtype", "tie_method"] + params = [ + ["float64", "float32", "int64", "datetime64"], + ["first", "average", "dense", "min", "max"], + ] + + def setup(self, dtype, tie_method): + N = 10**4 + if dtype == "datetime64": + data = np.array([Timestamp("2011/01/01")] * N, dtype=dtype) + else: + data = np.ones(N, dtype=dtype) + self.df = DataFrame({"values": data, "key": ["foo"] * N}) + + def time_rank_ties(self, dtype, tie_method): + self.df.groupby("key").rank(method=tie_method) + + +class Float32: + # GH 13335 + def setup(self): + tmp1 = (np.random.random(10000) * 0.1).astype(np.float32) + tmp2 = (np.random.random(10000) * 10.0).astype(np.float32) + tmp = np.concatenate((tmp1, tmp2)) + arr = np.repeat(tmp, 10) + self.df = DataFrame({"a": arr, "b": arr}) + + def time_sum(self): + self.df.groupby(["a"])["b"].sum() + + +class String: + # GH#41596 + param_names = ["dtype", "method"] + params = [ + ["str", "string[python]"], + [ + "sum", + "min", + "max", + "first", + "last", + "any", + "all", + ], + ] + + def setup(self, dtype, method): + cols = list("abcdefghjkl") + self.df = DataFrame( + np.random.randint(0, 100, size=(10_000, len(cols))), + columns=cols, + dtype=dtype, + ) + + def time_str_func(self, dtype, method): + self.df.groupby("a")[self.df.columns[1:]].agg(method) + + +class Categories: + params = [True, False] + param_names = ["observed"] + + def setup(self, observed): + N = 10**5 + arr = np.random.random(N) + data = {"a": Categorical(np.random.randint(10000, size=N)), "b": arr} + self.df = DataFrame(data) + data = { + "a": Categorical(np.random.randint(10000, size=N), ordered=True), + "b": arr, + } + self.df_ordered = DataFrame(data) + data = { + "a": Categorical( + np.random.randint(100, size=N), categories=np.arange(10000) + ), + "b": arr, + } + self.df_extra_cat = DataFrame(data) + + def time_groupby_sort(self, observed): + self.df.groupby("a", observed=observed)["b"].count() + + def time_groupby_nosort(self, observed): + self.df.groupby("a", observed=observed, sort=False)["b"].count() + + def time_groupby_ordered_sort(self, observed): + self.df_ordered.groupby("a", observed=observed)["b"].count() + + def time_groupby_ordered_nosort(self, observed): + self.df_ordered.groupby("a", observed=observed, sort=False)["b"].count() + + def time_groupby_extra_cat_sort(self, observed): + self.df_extra_cat.groupby("a", observed=observed)["b"].count() + + def time_groupby_extra_cat_nosort(self, observed): + self.df_extra_cat.groupby("a", observed=observed, sort=False)["b"].count() + + +class MultipleCategories: + def setup(self): + N = 10**3 + arr = np.random.random(N) + data = { + "a1": Categorical(np.random.randint(10000, size=N)), + "a2": Categorical(np.random.randint(10000, size=N)), + "b": arr, + } + self.df = DataFrame(data) + data = { + "a1": Categorical(np.random.randint(10000, size=N), ordered=True), + "a2": Categorical(np.random.randint(10000, size=N), ordered=True), + "b": arr, + } + self.df_ordered = DataFrame(data) + data = { + "a1": Categorical(np.random.randint(100, size=N), categories=np.arange(N)), + "a2": Categorical(np.random.randint(100, size=N), categories=np.arange(N)), + "b": arr, + } + self.df_extra_cat = DataFrame(data) + + def time_groupby_sort(self): + self.df.groupby(["a1", "a2"], observed=False)["b"].count() + + def time_groupby_nosort(self): + self.df.groupby(["a1", "a2"], observed=False, sort=False)["b"].count() + + def time_groupby_ordered_sort(self): + self.df_ordered.groupby(["a1", "a2"], observed=False)["b"].count() + + def time_groupby_ordered_nosort(self): + self.df_ordered.groupby(["a1", "a2"], observed=False, sort=False)["b"].count() + + def time_groupby_extra_cat_sort(self): + self.df_extra_cat.groupby(["a1", "a2"], observed=False)["b"].count() + + def time_groupby_extra_cat_nosort(self): + self.df_extra_cat.groupby(["a1", "a2"], observed=False, sort=False)["b"].count() + + def time_groupby_transform(self): + self.df_extra_cat.groupby(["a1", "a2"], observed=False)["b"].cumsum() + + +class Datelike: + # GH 14338 + params = ["period_range", "date_range", "date_range_tz"] + param_names = ["grouper"] + + def setup(self, grouper): + N = 10**4 + rng_map = { + "period_range": period_range, + "date_range": date_range, + "date_range_tz": partial(date_range, tz="US/Central"), + } + self.grouper = rng_map[grouper]("1900-01-01", freq="D", periods=N) + self.df = DataFrame(np.random.randn(10**4, 2)) + + def time_sum(self, grouper): + self.df.groupby(self.grouper).sum() + + +class SumBools: + # GH 2692 + def setup(self): + N = 500 + self.df = DataFrame({"ii": range(N), "bb": [True] * N}) + + def time_groupby_sum_booleans(self): + self.df.groupby("ii").sum() + + +class SumMultiLevel: + # GH 9049 + timeout = 120.0 + + def setup(self): + N = 50 + self.df = DataFrame( + {"A": list(range(N)) * 2, "B": range(N * 2), "C": 1} + ).set_index(["A", "B"]) + + def time_groupby_sum_multiindex(self): + self.df.groupby(level=[0, 1]).sum() + + +class SumTimeDelta: + # GH 20660 + def setup(self): + N = 10**4 + self.df = DataFrame( + np.random.randint(1000, 100000, (N, 100)), + index=np.random.randint(200, size=(N,)), + ).astype("timedelta64[ns]") + self.df_int = self.df.copy().astype("int64") + + def time_groupby_sum_timedelta(self): + self.df.groupby(lambda x: x).sum() + + def time_groupby_sum_int(self): + self.df_int.groupby(lambda x: x).sum() + + +class Transform: + def setup(self): + n1 = 400 + n2 = 250 + index = MultiIndex( + levels=[np.arange(n1), Index([f"i-{i}" for i in range(n2)], dtype=object)], + codes=[np.repeat(range(n1), n2).tolist(), list(range(n2)) * n1], + names=["lev1", "lev2"], + ) + arr = np.random.randn(n1 * n2, 3) + arr[::10000, 0] = np.nan + arr[1::10000, 1] = np.nan + arr[2::10000, 2] = np.nan + data = DataFrame(arr, index=index, columns=["col1", "col20", "col3"]) + self.df = data + + n = 1000 + self.df_wide = DataFrame( + np.random.randn(n, n), + index=np.random.choice(range(10), n), + ) + + n = 1_000_000 + self.df_tall = DataFrame( + np.random.randn(n, 3), + index=np.random.randint(0, 5, n), + ) + + n = 20000 + self.df1 = DataFrame( + np.random.randint(1, n, (n, 3)), columns=["jim", "joe", "jolie"] + ) + self.df2 = self.df1.copy() + self.df2["jim"] = self.df2["joe"] + + self.df3 = DataFrame( + np.random.randint(1, (n / 10), (n, 3)), columns=["jim", "joe", "jolie"] + ) + self.df4 = self.df3.copy() + self.df4["jim"] = self.df4["joe"] + + def time_transform_lambda_max(self): + self.df.groupby(level="lev1").transform(lambda x: max(x)) + + def time_transform_str_max(self): + self.df.groupby(level="lev1").transform("max") + + def time_transform_lambda_max_tall(self): + self.df_tall.groupby(level=0).transform(lambda x: np.max(x, axis=0)) + + def time_transform_lambda_max_wide(self): + self.df_wide.groupby(level=0).transform(lambda x: np.max(x, axis=0)) + + def time_transform_multi_key1(self): + self.df1.groupby(["jim", "joe"])["jolie"].transform("max") + + def time_transform_multi_key2(self): + self.df2.groupby(["jim", "joe"])["jolie"].transform("max") + + def time_transform_multi_key3(self): + self.df3.groupby(["jim", "joe"])["jolie"].transform("max") + + def time_transform_multi_key4(self): + self.df4.groupby(["jim", "joe"])["jolie"].transform("max") + + +class TransformBools: + def setup(self): + N = 120000 + transition_points = np.sort(np.random.choice(np.arange(N), 1400)) + transitions = np.zeros(N, dtype=np.bool_) + transitions[transition_points] = True + self.g = transitions.cumsum() + self.df = DataFrame({"signal": np.random.rand(N)}) + + def time_transform_mean(self): + self.df["signal"].groupby(self.g).transform("mean") + + +class TransformNaN: + # GH 12737 + def setup(self): + self.df_nans = DataFrame( + {"key": np.repeat(np.arange(1000), 10), "B": np.nan, "C": np.nan} + ) + self.df_nans.loc[4::10, "B":"C"] = 5 + + def time_first(self): + self.df_nans.groupby("key").transform("first") + + +class TransformEngine: + param_names = ["parallel"] + params = [[True, False]] + + def setup(self, parallel): + N = 10**3 + data = DataFrame( + {0: [str(i) for i in range(100)] * N, 1: list(range(100)) * N}, + columns=[0, 1], + ) + self.parallel = parallel + self.grouper = data.groupby(0) + + def time_series_numba(self, parallel): + def function(values, index): + return values * 5 + + self.grouper[1].transform( + function, engine="numba", engine_kwargs={"parallel": self.parallel} + ) + + def time_series_cython(self, parallel): + def function(values): + return values * 5 + + self.grouper[1].transform(function, engine="cython") + + def time_dataframe_numba(self, parallel): + def function(values, index): + return values * 5 + + self.grouper.transform( + function, engine="numba", engine_kwargs={"parallel": self.parallel} + ) + + def time_dataframe_cython(self, parallel): + def function(values): + return values * 5 + + self.grouper.transform(function, engine="cython") + + +class AggEngine: + param_names = ["parallel"] + params = [[True, False]] + + def setup(self, parallel): + N = 10**3 + data = DataFrame( + {0: [str(i) for i in range(100)] * N, 1: list(range(100)) * N}, + columns=[0, 1], + ) + self.parallel = parallel + self.grouper = data.groupby(0) + + def time_series_numba(self, parallel): + def function(values, index): + total = 0 + for i, value in enumerate(values): + if i % 2: + total += value + 5 + else: + total += value * 2 + return total + + self.grouper[1].agg( + function, engine="numba", engine_kwargs={"parallel": self.parallel} + ) + + def time_series_cython(self, parallel): + def function(values): + total = 0 + for i, value in enumerate(values): + if i % 2: + total += value + 5 + else: + total += value * 2 + return total + + self.grouper[1].agg(function, engine="cython") + + def time_dataframe_numba(self, parallel): + def function(values, index): + total = 0 + for i, value in enumerate(values): + if i % 2: + total += value + 5 + else: + total += value * 2 + return total + + self.grouper.agg( + function, engine="numba", engine_kwargs={"parallel": self.parallel} + ) + + def time_dataframe_cython(self, parallel): + def function(values): + total = 0 + for i, value in enumerate(values): + if i % 2: + total += value + 5 + else: + total += value * 2 + return total + + self.grouper.agg(function, engine="cython") + + +class Sample: + def setup(self): + N = 10**3 + self.df = DataFrame({"a": np.zeros(N)}) + self.groups = np.arange(0, N) + self.weights = np.ones(N) + + def time_sample(self): + self.df.groupby(self.groups).sample(n=1) + + def time_sample_weights(self): + self.df.groupby(self.groups).sample(n=1, weights=self.weights) + + +class Resample: + # GH 28635 + def setup(self): + num_timedeltas = 20_000 + num_groups = 3 + + index = MultiIndex.from_product( + [ + np.arange(num_groups), + to_timedelta(np.arange(num_timedeltas), unit="s"), + ], + names=["groups", "timedeltas"], + ) + data = np.random.randint(0, 1000, size=(len(index))) + + self.df = DataFrame(data, index=index).reset_index("timedeltas") + self.df_multiindex = DataFrame(data, index=index) + + def time_resample(self): + self.df.groupby(level="groups").resample("10s", on="timedeltas").mean() + + def time_resample_multiindex(self): + self.df_multiindex.groupby(level="groups").resample( + "10s", level="timedeltas" + ).mean() + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/hash_functions.py b/asv_bench/benchmarks/hash_functions.py new file mode 100644 index 0000000000000..d2c5b4dfbef70 --- /dev/null +++ b/asv_bench/benchmarks/hash_functions.py @@ -0,0 +1,89 @@ +import numpy as np + +import pandas as pd + + +class UniqueForLargePyObjectInts: + def setup(self): + lst = [x << 32 for x in range(5000)] + self.arr = np.array(lst, dtype=np.object_) + + def time_unique(self): + pd.unique(self.arr) + + +class Float64GroupIndex: + # GH28303 + def setup(self): + self.df = pd.date_range( + start="1/1/2018", end="1/2/2018", periods=10**6 + ).to_frame() + self.group_index = np.round(self.df.index.astype(int) / 10**9) + + def time_groupby(self): + self.df.groupby(self.group_index).last() + + +class UniqueAndFactorizeArange: + params = range(4, 16) + param_names = ["exponent"] + + def setup(self, exponent): + a = np.arange(10**4, dtype="float64") + self.a2 = (a + 10**exponent).repeat(100) + + def time_factorize(self, exponent): + pd.factorize(self.a2) + + def time_unique(self, exponent): + pd.unique(self.a2) + + +class Unique: + params = ["Int64", "Float64"] + param_names = ["dtype"] + + def setup(self, dtype): + self.ser = pd.Series(([1, pd.NA, 2] + list(range(100_000))) * 3, dtype=dtype) + self.ser_unique = pd.Series(list(range(300_000)) + [pd.NA], dtype=dtype) + + def time_unique_with_duplicates(self, exponent): + pd.unique(self.ser) + + def time_unique(self, exponent): + pd.unique(self.ser_unique) + + +class NumericSeriesIndexing: + params = [ + (np.int64, np.uint64, np.float64), + (10**4, 10**5, 5 * 10**5, 10**6, 5 * 10**6), + ] + param_names = ["dtype", "N"] + + def setup(self, dtype, N): + vals = np.array(list(range(55)) + [54] + list(range(55, N - 1)), dtype=dtype) + indices = pd.Index(vals) + self.data = pd.Series(np.arange(N), index=indices) + + def time_loc_slice(self, index, N): + # trigger building of mapping + self.data.loc[:800] + + +class NumericSeriesIndexingShuffled: + params = [ + (np.int64, np.uint64, np.float64), + (10**4, 10**5, 5 * 10**5, 10**6, 5 * 10**6), + ] + param_names = ["dtype", "N"] + + def setup(self, dtype, N): + vals = np.array(list(range(55)) + [54] + list(range(55, N - 1)), dtype=dtype) + np.random.shuffle(vals) + indices = pd.Index(vals) + self.data = pd.Series(np.arange(N), index=indices) + + def time_loc_slice(self, index, N): + # trigger building of mapping + self.data.loc[:800] diff --git a/asv_bench/benchmarks/index_cached_properties.py b/asv_bench/benchmarks/index_cached_properties.py new file mode 100644 index 0000000000000..d21bbe15c4cc8 --- /dev/null +++ b/asv_bench/benchmarks/index_cached_properties.py @@ -0,0 +1,72 @@ +import pandas as pd + + +class IndexCache: + number = 1 + repeat = (3, 100, 20) + + params = [ + [ + "CategoricalIndex", + "DatetimeIndex", + "Float64Index", + "IntervalIndex", + "Int64Index", + "MultiIndex", + "PeriodIndex", + "RangeIndex", + "TimedeltaIndex", + "UInt64Index", + ] + ] + param_names = ["index_type"] + + def setup(self, index_type): + N = 10**5 + if index_type == "MultiIndex": + self.idx = pd.MultiIndex.from_product( + [pd.date_range("1/1/2000", freq="min", periods=N // 2), ["a", "b"]] + ) + elif index_type == "DatetimeIndex": + self.idx = pd.date_range("1/1/2000", freq="min", periods=N) + elif index_type == "Int64Index": + self.idx = pd.Index(range(N), dtype="int64") + elif index_type == "PeriodIndex": + self.idx = pd.period_range("1/1/2000", freq="min", periods=N) + elif index_type == "RangeIndex": + self.idx = pd.RangeIndex(start=0, stop=N) + elif index_type == "IntervalIndex": + self.idx = pd.IntervalIndex.from_arrays(range(N), range(1, N + 1)) + elif index_type == "TimedeltaIndex": + self.idx = pd.TimedeltaIndex(range(N)) + elif index_type == "Float64Index": + self.idx = pd.Index(range(N), dtype="float64") + elif index_type == "UInt64Index": + self.idx = pd.Index(range(N), dtype="uint64") + elif index_type == "CategoricalIndex": + self.idx = pd.CategoricalIndex(range(N), range(N)) + else: + raise ValueError + assert len(self.idx) == N + self.idx._cache = {} + + def time_values(self, index_type): + self.idx._values + + def time_shape(self, index_type): + self.idx.shape + + def time_is_monotonic_decreasing(self, index_type): + self.idx.is_monotonic_decreasing + + def time_is_monotonic_increasing(self, index_type): + self.idx.is_monotonic_increasing + + def time_is_unique(self, index_type): + self.idx.is_unique + + def time_engine(self, index_type): + self.idx._engine + + def time_inferred_type(self, index_type): + self.idx.inferred_type diff --git a/asv_bench/benchmarks/index_object.py b/asv_bench/benchmarks/index_object.py new file mode 100644 index 0000000000000..9c1e9656503f7 --- /dev/null +++ b/asv_bench/benchmarks/index_object.py @@ -0,0 +1,262 @@ +import gc + +import numpy as np + +from pandas import ( + DatetimeIndex, + Index, + IntervalIndex, + MultiIndex, + RangeIndex, + Series, + date_range, +) + + +class SetOperations: + params = ( + ["monotonic", "non_monotonic"], + ["datetime", "date_string", "int", "strings", "ea_int"], + ["intersection", "union", "symmetric_difference"], + ) + param_names = ["index_structure", "dtype", "method"] + + def setup(self, index_structure, dtype, method): + N = 10**5 + dates_left = date_range("1/1/2000", periods=N, freq="min") + fmt = "%Y-%m-%d %H:%M:%S" + date_str_left = Index(dates_left.strftime(fmt)) + int_left = Index(np.arange(N)) + ea_int_left = Index(np.arange(N), dtype="Int64") + str_left = Index([f"i-{i}" for i in range(N)], dtype=object) + + data = { + "datetime": dates_left, + "date_string": date_str_left, + "int": int_left, + "strings": str_left, + "ea_int": ea_int_left, + } + + if index_structure == "non_monotonic": + data = {k: mi[::-1] for k, mi in data.items()} + + data = {k: {"left": idx, "right": idx[:-1]} for k, idx in data.items()} + + self.left = data[dtype]["left"] + self.right = data[dtype]["right"] + + def time_operation(self, index_structure, dtype, method): + getattr(self.left, method)(self.right) + + +class SetDisjoint: + def setup(self): + N = 10**5 + B = N + 20000 + self.datetime_left = DatetimeIndex(range(N)) + self.datetime_right = DatetimeIndex(range(N, B)) + + def time_datetime_difference_disjoint(self): + self.datetime_left.difference(self.datetime_right) + + +class UnionWithDuplicates: + def setup(self): + self.left = Index(np.repeat(np.arange(1000), 100)) + self.right = Index(np.tile(np.arange(500, 1500), 50)) + + def time_union_with_duplicates(self): + self.left.union(self.right) + + +class Range: + def setup(self): + self.idx_inc = RangeIndex(start=0, stop=10**6, step=3) + self.idx_dec = RangeIndex(start=10**6, stop=-1, step=-3) + + def time_max(self): + self.idx_inc.max() + + def time_max_trivial(self): + self.idx_dec.max() + + def time_min(self): + self.idx_dec.min() + + def time_min_trivial(self): + self.idx_inc.min() + + def time_get_loc_inc(self): + self.idx_inc.get_loc(900_000) + + def time_get_loc_dec(self): + self.idx_dec.get_loc(100_000) + + def time_iter_inc(self): + for _ in self.idx_inc: + pass + + def time_iter_dec(self): + for _ in self.idx_dec: + pass + + def time_sort_values_asc(self): + self.idx_inc.sort_values() + + def time_sort_values_des(self): + self.idx_inc.sort_values(ascending=False) + + +class IndexEquals: + def setup(self): + idx_large_fast = RangeIndex(100_000) + idx_small_slow = date_range(start="1/1/2012", periods=1) + self.mi_large_slow = MultiIndex.from_product([idx_large_fast, idx_small_slow]) + + self.idx_non_object = RangeIndex(1) + + def time_non_object_equals_multiindex(self): + self.idx_non_object.equals(self.mi_large_slow) + + +class IndexAppend: + def setup(self): + N = 10_000 + self.range_idx = RangeIndex(0, 100) + self.int_idx = self.range_idx.astype(int) + self.obj_idx = self.int_idx.astype(str) + self.range_idxs = [] + self.int_idxs = [] + self.object_idxs = [] + for i in range(1, N): + r_idx = RangeIndex(i * 100, (i + 1) * 100) + self.range_idxs.append(r_idx) + i_idx = r_idx.astype(int) + self.int_idxs.append(i_idx) + o_idx = i_idx.astype(str) + self.object_idxs.append(o_idx) + self.same_range_idx = [self.range_idx] * N + + def time_append_range_list(self): + self.range_idx.append(self.range_idxs) + + def time_append_range_list_same(self): + self.range_idx.append(self.same_range_idx) + + def time_append_int_list(self): + self.int_idx.append(self.int_idxs) + + def time_append_obj_list(self): + self.obj_idx.append(self.object_idxs) + + +class Indexing: + params = ["String", "Float", "Int"] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10**6 + if dtype == "String": + self.idx = Index([f"i-{i}" for i in range(N)], dtype=object) + elif dtype == "Float": + self.idx = Index(np.arange(N), dtype=np.float64) + elif dtype == "Int": + self.idx = Index(np.arange(N), dtype=np.int64) + self.array_mask = (np.arange(N) % 3) == 0 + self.series_mask = Series(self.array_mask) + self.sorted = self.idx.sort_values() + half = N // 2 + self.non_unique = self.idx[:half].append(self.idx[:half]) + self.non_unique_sorted = self.sorted[:half].repeat(2) + self.key = self.sorted[N // 4] + + def time_boolean_array(self, dtype): + self.idx[self.array_mask] + + def time_boolean_series(self, dtype): + self.idx[self.series_mask] + + def time_get(self, dtype): + self.idx[1] + + def time_slice(self, dtype): + self.idx[:-1] + + def time_slice_step(self, dtype): + self.idx[::2] + + def time_get_loc(self, dtype): + self.idx.get_loc(self.key) + + def time_get_loc_sorted(self, dtype): + self.sorted.get_loc(self.key) + + def time_get_loc_non_unique(self, dtype): + self.non_unique.get_loc(self.key) + + def time_get_loc_non_unique_sorted(self, dtype): + self.non_unique_sorted.get_loc(self.key) + + +class Float64IndexMethod: + # GH 13166 + def setup(self): + N = 100_000 + a = np.arange(N, dtype=np.float64) + self.ind = Index(a * 4.8000000418824129e-08) + + def time_get_loc(self): + self.ind.get_loc(0) + + +class IntervalIndexMethod: + # GH 24813 + params = [10**3, 10**5] + + def setup(self, N): + left = np.append(np.arange(N), np.array(0)) + right = np.append(np.arange(1, N + 1), np.array(1)) + self.intv = IntervalIndex.from_arrays(left, right) + self.intv._engine + + self.intv2 = IntervalIndex.from_arrays(left + 1, right + 1) + self.intv2._engine + + self.left = IntervalIndex.from_breaks(np.arange(N)) + self.right = IntervalIndex.from_breaks(np.arange(N - 3, 2 * N - 3)) + + def time_monotonic_inc(self, N): + self.intv.is_monotonic_increasing + + def time_is_unique(self, N): + self.intv.is_unique + + def time_intersection(self, N): + self.left.intersection(self.right) + + def time_intersection_one_duplicate(self, N): + self.intv.intersection(self.right) + + def time_intersection_both_duplicate(self, N): + self.intv.intersection(self.intv2) + + +class GC: + params = [1, 2, 5] + + def create_use_drop(self): + idx = Index(list(range(1_000_000))) + idx._engine + + def peakmem_gc_instances(self, N): + try: + gc.disable() + + for _ in range(N): + self.create_use_drop() + finally: + gc.enable() + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/indexing.py b/asv_bench/benchmarks/indexing.py new file mode 100644 index 0000000000000..b2495356f134c --- /dev/null +++ b/asv_bench/benchmarks/indexing.py @@ -0,0 +1,562 @@ +""" +These benchmarks are for Series and DataFrame indexing methods. For the +lower-level methods directly on Index and subclasses, see index_object.py, +indexing_engine.py, and index_cached.py +""" + +from datetime import datetime +import warnings + +import numpy as np + +from pandas import ( + NA, + CategoricalIndex, + DataFrame, + Index, + IntervalIndex, + MultiIndex, + Series, + concat, + date_range, + option_context, + period_range, +) + + +class NumericSeriesIndexing: + params = [ + (np.int64, np.uint64, np.float64), + ("unique_monotonic_inc", "nonunique_monotonic_inc"), + ] + param_names = ["dtype", "index_structure"] + + def setup(self, dtype, index_structure): + N = 10**6 + indices = { + "unique_monotonic_inc": Index(range(N), dtype=dtype), + "nonunique_monotonic_inc": Index( + list(range(55)) + [54] + list(range(55, N - 1)), dtype=dtype + ), + } + self.data = Series(np.random.rand(N), index=indices[index_structure]) + self.array = np.arange(10000) + self.array_list = self.array.tolist() + + def time_getitem_scalar(self, index, index_structure): + self.data[800000] + + def time_getitem_slice(self, index, index_structure): + self.data[:800000] + + def time_getitem_list_like(self, index, index_structure): + self.data[[800000]] + + def time_getitem_array(self, index, index_structure): + self.data[self.array] + + def time_getitem_lists(self, index, index_structure): + self.data[self.array_list] + + def time_iloc_array(self, index, index_structure): + self.data.iloc[self.array] + + def time_iloc_list_like(self, index, index_structure): + self.data.iloc[[800000]] + + def time_iloc_scalar(self, index, index_structure): + self.data.iloc[800000] + + def time_iloc_slice(self, index, index_structure): + self.data.iloc[:800000] + + def time_loc_array(self, index, index_structure): + self.data.loc[self.array] + + def time_loc_list_like(self, index, index_structure): + self.data.loc[[800000]] + + def time_loc_scalar(self, index, index_structure): + self.data.loc[800000] + + def time_loc_slice(self, index, index_structure): + self.data.loc[:800000] + + +class NumericMaskedIndexing: + monotonic_list = list(range(10**6)) + non_monotonic_list = list(range(50)) + [54, 53, 52, 51] + list(range(55, 10**6 - 1)) + + params = [ + ("Int64", "UInt64", "Float64"), + (True, False), + ] + param_names = ["dtype", "monotonic"] + + def setup(self, dtype, monotonic): + indices = { + True: Index(self.monotonic_list, dtype=dtype), + False: Index(self.non_monotonic_list, dtype=dtype).append( + Index([NA], dtype=dtype) + ), + } + self.data = indices[monotonic] + self.indexer = np.arange(300, 1_000) + self.data_dups = self.data.append(self.data) + + def time_get_indexer(self, dtype, monotonic): + self.data.get_indexer(self.indexer) + + def time_get_indexer_dups(self, dtype, monotonic): + self.data.get_indexer_for(self.indexer) + + +class NonNumericSeriesIndexing: + params = [ + ("string", "datetime", "period"), + ("unique_monotonic_inc", "nonunique_monotonic_inc", "non_monotonic"), + ] + param_names = ["index_dtype", "index_structure"] + + def setup(self, index, index_structure): + N = 10**6 + if index == "string": + index = Index([f"i-{i}" for i in range(N)], dtype=object) + elif index == "datetime": + index = date_range("1900", periods=N, freq="s") + elif index == "period": + index = period_range("1900", periods=N, freq="s") + index = index.sort_values() + assert index.is_unique and index.is_monotonic_increasing + if index_structure == "nonunique_monotonic_inc": + index = index.insert(item=index[2], loc=2)[:-1] + elif index_structure == "non_monotonic": + index = index[::2].append(index[1::2]) + assert len(index) == N + self.s = Series(np.random.rand(N), index=index) + self.lbl = index[80000] + # warm up index mapping + self.s[self.lbl] + + def time_getitem_label_slice(self, index, index_structure): + self.s[: self.lbl] + + def time_getitem_pos_slice(self, index, index_structure): + self.s[:80000] + + def time_getitem_scalar(self, index, index_structure): + self.s[self.lbl] + + def time_getitem_list_like(self, index, index_structure): + self.s[[self.lbl]] + + +class DataFrameStringIndexing: + def setup(self): + index = Index([f"i-{i}" for i in range(1000)], dtype=object) + columns = Index([f"i-{i}" for i in range(30)], dtype=object) + with warnings.catch_warnings(record=True): + self.df = DataFrame(np.random.randn(1000, 30), index=index, columns=columns) + self.idx_scalar = index[100] + self.col_scalar = columns[10] + self.bool_indexer = self.df[self.col_scalar] > 0 + self.bool_obj_indexer = self.bool_indexer.astype(object) + self.boolean_indexer = (self.df[self.col_scalar] > 0).astype("boolean") + + def time_loc(self): + self.df.loc[self.idx_scalar, self.col_scalar] + + def time_at(self): + self.df.at[self.idx_scalar, self.col_scalar] + + def time_at_setitem(self): + self.df.at[self.idx_scalar, self.col_scalar] = 0.0 + + def time_getitem_scalar(self): + self.df[self.col_scalar][self.idx_scalar] + + def time_boolean_rows(self): + self.df[self.bool_indexer] + + def time_boolean_rows_object(self): + self.df[self.bool_obj_indexer] + + def time_boolean_rows_boolean(self): + self.df[self.boolean_indexer] + + +class DataFrameNumericIndexing: + params = [ + (np.int64, np.uint64, np.float64), + ("unique_monotonic_inc", "nonunique_monotonic_inc"), + ] + param_names = ["dtype", "index_structure"] + + def setup(self, dtype, index_structure): + N = 10**5 + indices = { + "unique_monotonic_inc": Index(range(N), dtype=dtype), + "nonunique_monotonic_inc": Index( + list(range(55)) + [54] + list(range(55, N - 1)), dtype=dtype + ), + } + self.idx_dupe = np.array(range(30)) * 99 + self.df = DataFrame(np.random.randn(N, 5), index=indices[index_structure]) + self.df_dup = concat([self.df, 2 * self.df, 3 * self.df]) + self.bool_indexer = [True] * (N // 2) + [False] * (N - N // 2) + + def time_iloc_dups(self, index, index_structure): + self.df_dup.iloc[self.idx_dupe] + + def time_loc_dups(self, index, index_structure): + self.df_dup.loc[self.idx_dupe] + + def time_iloc(self, index, index_structure): + self.df.iloc[:100, 0] + + def time_loc(self, index, index_structure): + self.df.loc[:100, 0] + + def time_bool_indexer(self, index, index_structure): + self.df[self.bool_indexer] + + +class Take: + params = ["int", "datetime"] + param_names = ["index"] + + def setup(self, index): + N = 100000 + indexes = { + "int": Index(np.arange(N), dtype=np.int64), + "datetime": date_range("2011-01-01", freq="s", periods=N), + } + index = indexes[index] + self.s = Series(np.random.rand(N), index=index) + self.indexer = np.random.randint(0, N, size=N) + + def time_take(self, index): + self.s.take(self.indexer) + + +class MultiIndexing: + params = [True, False] + param_names = ["unique_levels"] + + def setup(self, unique_levels): + self.nlevels = 2 + if unique_levels: + mi = MultiIndex.from_arrays([range(1000000)] * self.nlevels) + else: + mi = MultiIndex.from_product([range(1000)] * self.nlevels) + self.df = DataFrame(np.random.randn(len(mi)), index=mi) + + self.tgt_slice = slice(200, 800) + self.tgt_null_slice = slice(None) + self.tgt_list = list(range(0, 1000, 10)) + self.tgt_scalar = 500 + + bool_indexer = np.zeros(len(mi), dtype=np.bool_) + bool_indexer[slice(0, len(mi), 100)] = True + self.tgt_bool_indexer = bool_indexer + + def time_loc_partial_key_slice(self, unique_levels): + self.df.loc[self.tgt_slice, :] + + def time_loc_partial_key_null_slice(self, unique_levels): + self.df.loc[self.tgt_null_slice, :] + + def time_loc_partial_key_list(self, unique_levels): + self.df.loc[self.tgt_list, :] + + def time_loc_partial_key_scalar(self, unique_levels): + self.df.loc[self.tgt_scalar, :] + + def time_loc_partial_key_bool_indexer(self, unique_levels): + self.df.loc[self.tgt_bool_indexer, :] + + def time_loc_all_slices(self, unique_levels): + target = tuple([self.tgt_slice] * self.nlevels) + self.df.loc[target, :] + + def time_loc_all_null_slices(self, unique_levels): + target = tuple([self.tgt_null_slice] * self.nlevels) + self.df.loc[target, :] + + def time_loc_all_lists(self, unique_levels): + target = tuple([self.tgt_list] * self.nlevels) + self.df.loc[target, :] + + def time_loc_all_scalars(self, unique_levels): + target = tuple([self.tgt_scalar] * self.nlevels) + self.df.loc[target, :] + + def time_loc_all_bool_indexers(self, unique_levels): + target = tuple([self.tgt_bool_indexer] * self.nlevels) + self.df.loc[target, :] + + def time_loc_slice_plus_null_slice(self, unique_levels): + target = (self.tgt_slice, self.tgt_null_slice) + self.df.loc[target, :] + + def time_loc_null_slice_plus_slice(self, unique_levels): + target = (self.tgt_null_slice, self.tgt_slice) + self.df.loc[target, :] + + def time_loc_multiindex(self, unique_levels): + target = self.df.index[::10] + self.df.loc[target] + + def time_xs_level_0(self, unique_levels): + target = self.tgt_scalar + self.df.xs(target, level=0) + + def time_xs_level_1(self, unique_levels): + target = self.tgt_scalar + self.df.xs(target, level=1) + + def time_xs_full_key(self, unique_levels): + target = tuple([self.tgt_scalar] * self.nlevels) + self.df.xs(target) + + +class IntervalIndexing: + def setup_cache(self): + idx = IntervalIndex.from_breaks(np.arange(1000001)) + monotonic = Series(np.arange(1000000), index=idx) + return monotonic + + def time_getitem_scalar(self, monotonic): + monotonic[80000] + + def time_loc_scalar(self, monotonic): + monotonic.loc[80000] + + def time_getitem_list(self, monotonic): + monotonic[80000:] + + def time_loc_list(self, monotonic): + monotonic.loc[80000:] + + +class DatetimeIndexIndexing: + def setup(self): + dti = date_range("2016-01-01", periods=10000, tz="US/Pacific") + dti2 = dti.tz_convert("UTC") + self.dti = dti + self.dti2 = dti2 + + def time_get_indexer_mismatched_tz(self): + # reached via e.g. + # ser = Series(range(len(dti)), index=dti) + # ser[dti2] + self.dti.get_indexer(self.dti2) + + +class SortedAndUnsortedDatetimeIndexLoc: + def setup(self): + dti = date_range("2016-01-01", periods=10000, tz="US/Pacific") + index = np.array(dti) + + unsorted_index = index.copy() + unsorted_index[10] = unsorted_index[20] + + self.df_unsorted = DataFrame(index=unsorted_index, data={"a": 1}) + self.df_sort = DataFrame(index=index, data={"a": 1}) + + def time_loc_unsorted(self): + self.df_unsorted.loc["2016-6-11"] + + def time_loc_sorted(self): + self.df_sort.loc["2016-6-11"] + + +class CategoricalIndexIndexing: + params = ["monotonic_incr", "monotonic_decr", "non_monotonic"] + param_names = ["index"] + + def setup(self, index): + N = 10**5 + values = list("a" * N + "b" * N + "c" * N) + indices = { + "monotonic_incr": CategoricalIndex(values), + "monotonic_decr": CategoricalIndex(reversed(values)), + "non_monotonic": CategoricalIndex(list("abc" * N)), + } + self.data = indices[index] + self.data_unique = CategoricalIndex([str(i) for i in range(N * 3)]) + + self.int_scalar = 10000 + self.int_list = list(range(10000)) + + self.cat_scalar = "b" + self.cat_list = ["1", "3"] + + def time_getitem_scalar(self, index): + self.data[self.int_scalar] + + def time_getitem_slice(self, index): + self.data[: self.int_scalar] + + def time_getitem_list_like(self, index): + self.data[[self.int_scalar]] + + def time_getitem_list(self, index): + self.data[self.int_list] + + def time_getitem_bool_array(self, index): + self.data[self.data == self.cat_scalar] + + def time_get_loc_scalar(self, index): + self.data.get_loc(self.cat_scalar) + + def time_get_indexer_list(self, index): + self.data_unique.get_indexer(self.cat_list) + + +class MethodLookup: + def setup_cache(self): + s = Series() + return s + + def time_lookup_iloc(self, s): + s.iloc + + def time_lookup_loc(self, s): + s.loc + + +class GetItemSingleColumn: + def setup(self): + self.df_string_col = DataFrame(np.random.randn(3000, 1), columns=["A"]) + self.df_int_col = DataFrame(np.random.randn(3000, 1)) + + def time_frame_getitem_single_column_label(self): + self.df_string_col["A"] + + def time_frame_getitem_single_column_int(self): + self.df_int_col[0] + + +class IndexSingleRow: + params = [True, False] + param_names = ["unique_cols"] + + def setup(self, unique_cols): + arr = np.arange(10**7).reshape(-1, 10) + df = DataFrame(arr) + dtypes = ["u1", "u2", "u4", "u8", "i1", "i2", "i4", "i8", "f8", "f4"] + for i, d in enumerate(dtypes): + df[i] = df[i].astype(d) + + if not unique_cols: + # GH#33032 single-row lookups with non-unique columns were + # 15x slower than with unique columns + df.columns = ["A", "A"] + list(df.columns[2:]) + + self.df = df + + def time_iloc_row(self, unique_cols): + self.df.iloc[10000] + + def time_loc_row(self, unique_cols): + self.df.loc[10000] + + +class AssignTimeseriesIndex: + def setup(self): + N = 100000 + idx = date_range("1/1/2000", periods=N, freq="h") + self.df = DataFrame(np.random.randn(N, 1), columns=["A"], index=idx) + + def time_frame_assign_timeseries_index(self): + self.df["date"] = self.df.index + + +class InsertColumns: + def setup(self): + self.N = 10**3 + self.df = DataFrame(index=range(self.N)) + self.df2 = DataFrame(np.random.randn(self.N, 2)) + + def time_insert(self): + for i in range(100): + self.df.insert(0, i, np.random.randn(self.N), allow_duplicates=True) + + def time_insert_middle(self): + # same as time_insert but inserting to a middle column rather than + # front or back (which have fast-paths) + for i in range(100): + self.df2.insert( + 1, "colname", np.random.randn(self.N), allow_duplicates=True + ) + + def time_assign_with_setitem(self): + for i in range(100): + self.df[i] = np.random.randn(self.N) + + def time_assign_list_like_with_setitem(self): + self.df[list(range(100))] = np.random.randn(self.N, 100) + + def time_assign_list_of_columns_concat(self): + df = DataFrame(np.random.randn(self.N, 100)) + concat([self.df, df], axis=1) + + +class Setitem: + def setup(self): + N = 500_000 + cols = 500 + self.df = DataFrame(np.random.rand(N, cols)) + + def time_setitem(self): + self.df[100] = 100 + + def time_setitem_list(self): + self.df[[100, 200, 300]] = 100 + + +class SetitemObjectDtype: + # GH#19299 + + def setup(self): + N = 1000 + cols = 500 + self.df = DataFrame(index=range(N), columns=range(cols), dtype=object) + + def time_setitem_object_dtype(self): + self.df.loc[0, 1] = 1.0 + + +class ChainIndexing: + params = [None, "warn"] + param_names = ["mode"] + + def setup(self, mode): + self.N = 1000000 + self.df = DataFrame({"A": np.arange(self.N), "B": "foo"}) + + def time_chained_indexing(self, mode): + df = self.df + N = self.N + with warnings.catch_warnings(record=True): + with option_context("mode.chained_assignment", mode): + df2 = df[df.A > N // 2] + df2["C"] = 1.0 + + +class Block: + def setup(self): + self.df = DataFrame( + False, + columns=np.arange(500).astype(str), + index=date_range("2010-01-01", "2011-01-01"), + ) + + def time_test(self): + start = datetime(2010, 5, 1) + end = datetime(2010, 9, 1) + self.df.loc[start:end, :] = True + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/indexing_engines.py b/asv_bench/benchmarks/indexing_engines.py new file mode 100644 index 0000000000000..da0e7de585391 --- /dev/null +++ b/asv_bench/benchmarks/indexing_engines.py @@ -0,0 +1,192 @@ +""" +Benchmarks in this file depend mostly on code in _libs/ + +We have to created masked arrays to test the masked engine though. The +array is unpacked on the Cython level. + +If a PR does not edit anything in _libs, it is very unlikely that benchmarks +in this file will be affected. +""" + +import numpy as np + +from pandas._libs import index as libindex + +from pandas.core.arrays import BaseMaskedArray + + +def _get_numeric_engines(): + engine_names = [ + ("Int64Engine", np.int64), + ("Int32Engine", np.int32), + ("Int16Engine", np.int16), + ("Int8Engine", np.int8), + ("UInt64Engine", np.uint64), + ("UInt32Engine", np.uint32), + ("UInt16engine", np.uint16), + ("UInt8Engine", np.uint8), + ("Float64Engine", np.float64), + ("Float32Engine", np.float32), + ] + return [ + (getattr(libindex, engine_name), dtype) + for engine_name, dtype in engine_names + if hasattr(libindex, engine_name) + ] + + +def _get_masked_engines(): + engine_names = [ + ("MaskedInt64Engine", "Int64"), + ("MaskedInt32Engine", "Int32"), + ("MaskedInt16Engine", "Int16"), + ("MaskedInt8Engine", "Int8"), + ("MaskedUInt64Engine", "UInt64"), + ("MaskedUInt32Engine", "UInt32"), + ("MaskedUInt16engine", "UInt16"), + ("MaskedUInt8Engine", "UInt8"), + ("MaskedFloat64Engine", "Float64"), + ("MaskedFloat32Engine", "Float32"), + ] + return [ + (getattr(libindex, engine_name), dtype) + for engine_name, dtype in engine_names + if hasattr(libindex, engine_name) + ] + + +class NumericEngineIndexing: + params = [ + _get_numeric_engines(), + ["monotonic_incr", "monotonic_decr", "non_monotonic"], + [True, False], + [10**5, 2 * 10**6], # 2e6 is above SIZE_CUTOFF + ] + param_names = ["engine_and_dtype", "index_type", "unique", "N"] + + def setup(self, engine_and_dtype, index_type, unique, N): + engine, dtype = engine_and_dtype + + if ( + index_type == "non_monotonic" + and dtype in [np.int16, np.int8, np.uint8] + and unique + ): + # Values overflow + raise NotImplementedError + + if index_type == "monotonic_incr": + if unique: + arr = np.arange(N * 3, dtype=dtype) + else: + arr = np.array([1, 2, 3], dtype=dtype).repeat(N) + elif index_type == "monotonic_decr": + if unique: + arr = np.arange(N * 3, dtype=dtype)[::-1] + else: + arr = np.array([3, 2, 1], dtype=dtype).repeat(N) + else: + assert index_type == "non_monotonic" + if unique: + arr = np.empty(N * 3, dtype=dtype) + arr[:N] = np.arange(N * 2, N * 3, dtype=dtype) + arr[N:] = np.arange(N * 2, dtype=dtype) + else: + arr = np.array([1, 2, 3], dtype=dtype).repeat(N) + + self.data = engine(arr) + # code below avoids populating the mapping etc. while timing. + self.data.get_loc(2) + + self.key_middle = arr[len(arr) // 2] + self.key_early = arr[2] + + def time_get_loc(self, engine_and_dtype, index_type, unique, N): + self.data.get_loc(self.key_early) + + def time_get_loc_near_middle(self, engine_and_dtype, index_type, unique, N): + # searchsorted performance may be different near the middle of a range + # vs near an endpoint + self.data.get_loc(self.key_middle) + + +class MaskedNumericEngineIndexing: + params = [ + _get_masked_engines(), + ["monotonic_incr", "monotonic_decr", "non_monotonic"], + [True, False], + [10**5, 2 * 10**6], # 2e6 is above SIZE_CUTOFF + ] + param_names = ["engine_and_dtype", "index_type", "unique", "N"] + + def setup(self, engine_and_dtype, index_type, unique, N): + engine, dtype = engine_and_dtype + dtype = dtype.lower() + + if ( + index_type == "non_monotonic" + and dtype in ["int16", "int8", "uint8"] + and unique + ): + # Values overflow + raise NotImplementedError + + if index_type == "monotonic_incr": + if unique: + arr = np.arange(N * 3, dtype=dtype) + else: + arr = np.array([1, 2, 3], dtype=dtype).repeat(N) + mask = np.zeros(N * 3, dtype=np.bool_) + elif index_type == "monotonic_decr": + if unique: + arr = np.arange(N * 3, dtype=dtype)[::-1] + else: + arr = np.array([3, 2, 1], dtype=dtype).repeat(N) + mask = np.zeros(N * 3, dtype=np.bool_) + else: + assert index_type == "non_monotonic" + if unique: + arr = np.zeros(N * 3, dtype=dtype) + arr[:N] = np.arange(N * 2, N * 3, dtype=dtype) + arr[N:] = np.arange(N * 2, dtype=dtype) + + else: + arr = np.array([1, 2, 3], dtype=dtype).repeat(N) + mask = np.zeros(N * 3, dtype=np.bool_) + mask[-1] = True + + self.data = engine(BaseMaskedArray(arr, mask)) + # code below avoids populating the mapping etc. while timing. + self.data.get_loc(2) + + self.key_middle = arr[len(arr) // 2] + self.key_early = arr[2] + + def time_get_loc(self, engine_and_dtype, index_type, unique, N): + self.data.get_loc(self.key_early) + + def time_get_loc_near_middle(self, engine_and_dtype, index_type, unique, N): + # searchsorted performance may be different near the middle of a range + # vs near an endpoint + self.data.get_loc(self.key_middle) + + +class ObjectEngineIndexing: + params = [("monotonic_incr", "monotonic_decr", "non_monotonic")] + param_names = ["index_type"] + + def setup(self, index_type): + N = 10**5 + values = list("a" * N + "b" * N + "c" * N) + arr = { + "monotonic_incr": np.array(values, dtype=object), + "monotonic_decr": np.array(list(reversed(values)), dtype=object), + "non_monotonic": np.array(list("abc") * N, dtype=object), + }[index_type] + + self.data = libindex.ObjectEngine(arr) + # code below avoids populating the mapping etc. while timing. + self.data.get_loc("b") + + def time_get_loc(self, index_type): + self.data.get_loc("b") diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py new file mode 100644 index 0000000000000..ce3935d2cd0ac --- /dev/null +++ b/asv_bench/benchmarks/inference.py @@ -0,0 +1,297 @@ +""" +The functions benchmarked in this file depend _almost_ exclusively on +_libs, but not in a way that is easy to formalize. + +If a PR does not change anything in pandas/_libs/ or pandas/core/tools/, then +it is likely that these benchmarks will be unaffected. +""" + +import numpy as np + +from pandas import ( + Index, + NaT, + Series, + date_range, + to_datetime, + to_numeric, + to_timedelta, +) + +from .pandas_vb_common import lib + + +class ToNumeric: + def setup(self): + N = 10000 + self.float = Series(np.random.randn(N)) + self.numstr = self.float.astype("str") + self.str = Series(Index([f"i-{i}" for i in range(N)], dtype=object)) + + def time_from_float(self): + to_numeric(self.float, errors="coerce") + + def time_from_numeric_str(self): + to_numeric(self.numstr, errors="coerce") + + def time_from_str(self): + to_numeric(self.str, errors="coerce") + + +class ToNumericDowncast: + param_names = ["dtype", "downcast"] + params = [ + [ + "string-float", + "string-int", + "string-nint", + "datetime64", + "int-list", + "int32", + ], + [None, "integer", "signed", "unsigned", "float"], + ] + + N = 500000 + N2 = N // 2 + + data_dict = { + "string-int": ["1"] * N2 + [2] * N2, + "string-nint": ["-1"] * N2 + [2] * N2, + "datetime64": np.repeat( + np.array(["1970-01-01", "1970-01-02"], dtype="datetime64[D]"), N + ), + "string-float": ["1.1"] * N2 + [2] * N2, + "int-list": [1] * N2 + [2] * N2, + "int32": np.repeat(np.int32(1), N), + } + + def setup(self, dtype, downcast): + self.data = self.data_dict[dtype] + + def time_downcast(self, dtype, downcast): + to_numeric(self.data, downcast=downcast) + + +class MaybeConvertNumeric: + # maybe_convert_numeric depends _exclusively_ on _libs, could + # go in benchmarks/libs.py + + def setup_cache(self): + N = 10**6 + arr = np.repeat([2**63], N) + np.arange(N).astype("uint64") + data = arr.astype(object) + data[1::2] = arr[1::2].astype(str) + data[-1] = -1 + return data + + def time_convert(self, data): + lib.maybe_convert_numeric(data, set(), coerce_numeric=False) + + +class MaybeConvertObjects: + # maybe_convert_objects depends _almost_ exclusively on _libs, but + # does have some run-time imports from outside of _libs + + def setup(self): + N = 10**5 + + data = list(range(N)) + data[0] = NaT + data = np.array(data) + self.data = data + + def time_maybe_convert_objects(self): + lib.maybe_convert_objects(self.data) + + +class ToDatetimeFromIntsFloats: + def setup(self): + self.ts_sec = Series(range(1521080307, 1521685107), dtype="int64") + self.ts_sec_uint = Series(range(1521080307, 1521685107), dtype="uint64") + self.ts_sec_float = self.ts_sec.astype("float64") + + self.ts_nanosec = 1_000_000 * self.ts_sec + self.ts_nanosec_uint = 1_000_000 * self.ts_sec_uint + self.ts_nanosec_float = self.ts_nanosec.astype("float64") + + # speed of int64, uint64 and float64 paths should be comparable + + def time_nanosec_int64(self): + to_datetime(self.ts_nanosec, unit="ns") + + def time_nanosec_uint64(self): + to_datetime(self.ts_nanosec_uint, unit="ns") + + def time_nanosec_float64(self): + to_datetime(self.ts_nanosec_float, unit="ns") + + def time_sec_uint64(self): + to_datetime(self.ts_sec_uint, unit="s") + + def time_sec_int64(self): + to_datetime(self.ts_sec, unit="s") + + def time_sec_float64(self): + to_datetime(self.ts_sec_float, unit="s") + + +class ToDatetimeYYYYMMDD: + def setup(self): + rng = date_range(start="1/1/2000", periods=10000, freq="D") + self.stringsD = Series(rng.strftime("%Y%m%d")) + + def time_format_YYYYMMDD(self): + to_datetime(self.stringsD, format="%Y%m%d") + + +class ToDatetimeCacheSmallCount: + params = ([True, False], [50, 500, 5000, 100000]) + param_names = ["cache", "count"] + + def setup(self, cache, count): + rng = date_range(start="1/1/1971", periods=count) + self.unique_date_strings = rng.strftime("%Y-%m-%d").tolist() + + def time_unique_date_strings(self, cache, count): + to_datetime(self.unique_date_strings, cache=cache) + + +class ToDatetimeISO8601: + def setup(self): + rng = date_range(start="1/1/2000", periods=20000, freq="h") + self.strings = rng.strftime("%Y-%m-%d %H:%M:%S").tolist() + self.strings_nosep = rng.strftime("%Y%m%d %H:%M:%S").tolist() + self.strings_tz_space = [ + x.strftime("%Y-%m-%d %H:%M:%S") + " -0800" for x in rng + ] + self.strings_zero_tz = [x.strftime("%Y-%m-%d %H:%M:%S") + "Z" for x in rng] + + def time_iso8601(self): + to_datetime(self.strings) + + def time_iso8601_nosep(self): + to_datetime(self.strings_nosep) + + def time_iso8601_format(self): + to_datetime(self.strings, format="%Y-%m-%d %H:%M:%S") + + def time_iso8601_format_no_sep(self): + to_datetime(self.strings_nosep, format="%Y%m%d %H:%M:%S") + + def time_iso8601_tz_spaceformat(self): + to_datetime(self.strings_tz_space) + + def time_iso8601_infer_zero_tz_fromat(self): + # GH 41047 + to_datetime(self.strings_zero_tz) + + +class ToDatetimeNONISO8601: + def setup(self): + N = 10000 + half = N // 2 + ts_string_1 = "March 1, 2018 12:00:00+0400" + ts_string_2 = "March 1, 2018 12:00:00+0500" + self.same_offset = [ts_string_1] * N + self.diff_offset = [ts_string_1] * half + [ts_string_2] * half + + def time_same_offset(self): + to_datetime(self.same_offset) + + def time_different_offset(self): + to_datetime(self.diff_offset, utc=True) + + +class ToDatetimeFormatQuarters: + def setup(self): + self.s = Series(["2Q2005", "2Q05", "2005Q1", "05Q1"] * 10000) + + def time_infer_quarter(self): + to_datetime(self.s) + + +class ToDatetimeFormat: + def setup(self): + N = 100000 + self.s = Series(["19MAY11", "19MAY11:00:00:00"] * N) + self.s2 = self.s.str.replace(":\\S+$", "", regex=True) + + self.same_offset = ["10/11/2018 00:00:00.045-07:00"] * N + self.diff_offset = [ + f"10/11/2018 00:00:00.045-0{offset}:00" for offset in range(10) + ] * (N // 10) + + def time_exact(self): + to_datetime(self.s2, format="%d%b%y") + + def time_no_exact(self): + to_datetime(self.s, format="%d%b%y", exact=False) + + def time_same_offset(self): + to_datetime(self.same_offset, format="%m/%d/%Y %H:%M:%S.%f%z") + + def time_same_offset_to_utc(self): + to_datetime(self.same_offset, format="%m/%d/%Y %H:%M:%S.%f%z", utc=True) + + def time_different_offset_to_utc(self): + to_datetime(self.diff_offset, format="%m/%d/%Y %H:%M:%S.%f%z", utc=True) + + +class ToDatetimeCache: + params = [True, False] + param_names = ["cache"] + + def setup(self, cache): + N = 10000 + self.unique_numeric_seconds = list(range(N)) + self.dup_numeric_seconds = [1000] * N + self.dup_string_dates = ["2000-02-11"] * N + self.dup_string_with_tz = ["2000-02-11 15:00:00-0800"] * N + + def time_unique_seconds_and_unit(self, cache): + to_datetime(self.unique_numeric_seconds, unit="s", cache=cache) + + def time_dup_seconds_and_unit(self, cache): + to_datetime(self.dup_numeric_seconds, unit="s", cache=cache) + + def time_dup_string_dates(self, cache): + to_datetime(self.dup_string_dates, cache=cache) + + def time_dup_string_dates_and_format(self, cache): + to_datetime(self.dup_string_dates, format="%Y-%m-%d", cache=cache) + + def time_dup_string_tzoffset_dates(self, cache): + to_datetime(self.dup_string_with_tz, cache=cache) + + +class ToTimedelta: + def setup(self): + self.ints = np.random.randint(0, 60, size=10000) + self.str_days = [] + self.str_seconds = [] + for i in self.ints: + self.str_days.append(f"{i} days") + self.str_seconds.append(f"00:00:{i:02d}") + + def time_convert_int(self): + to_timedelta(self.ints, unit="s") + + def time_convert_string_days(self): + to_timedelta(self.str_days) + + def time_convert_string_seconds(self): + to_timedelta(self.str_seconds) + + +class ToTimedeltaErrors: + def setup(self): + ints = np.random.randint(0, 60, size=10000) + self.arr = [f"{i} days" for i in ints] + self.arr[-1] = "apple" + + def time_convert(self): + to_timedelta(self.arr, errors="coerce") + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/doc/sphinxext/ipython_sphinxext/__init__.py b/asv_bench/benchmarks/io/__init__.py similarity index 100% rename from doc/sphinxext/ipython_sphinxext/__init__.py rename to asv_bench/benchmarks/io/__init__.py diff --git a/asv_bench/benchmarks/io/csv.py b/asv_bench/benchmarks/io/csv.py new file mode 100644 index 0000000000000..9ee867260aa39 --- /dev/null +++ b/asv_bench/benchmarks/io/csv.py @@ -0,0 +1,645 @@ +from io import ( + BytesIO, + StringIO, +) +import random +import string + +import numpy as np + +from pandas import ( + Categorical, + DataFrame, + Index, + concat, + date_range, + period_range, + read_csv, + to_datetime, +) + +from ..pandas_vb_common import BaseIO + + +class ToCSV(BaseIO): + fname = "__test__.csv" + params = ["wide", "long", "mixed"] + param_names = ["kind"] + + def setup(self, kind): + wide_frame = DataFrame(np.random.randn(3000, 30)) + long_frame = DataFrame( + { + "A": np.arange(50000), + "B": np.arange(50000) + 1.0, + "C": np.arange(50000) + 2.0, + "D": np.arange(50000) + 3.0, + } + ) + mixed_frame = DataFrame( + { + "float": np.random.randn(5000), + "int": np.random.randn(5000).astype(int), + "bool": (np.arange(5000) % 2) == 0, + "datetime": date_range("2001", freq="s", periods=5000), + "object": ["foo"] * 5000, + } + ) + mixed_frame.loc[30:500, "float"] = np.nan + data = {"wide": wide_frame, "long": long_frame, "mixed": mixed_frame} + self.df = data[kind] + + def time_frame(self, kind): + self.df.to_csv(self.fname) + + +class ToCSVFloatFormatVariants(BaseIO): + fname = "__test__.csv" + + def setup(self): + self.df = DataFrame(np.random.default_rng(seed=42).random((1000, 1000))) + + def time_old_style_percent_format(self): + self.df.to_csv(self.fname, float_format="%.6f") + + def time_new_style_brace_format(self): + self.df.to_csv(self.fname, float_format="{:.6f}") + + def time_new_style_thousands_format(self): + self.df.to_csv(self.fname, float_format="{:,.2f}") + + def time_callable_format(self): + self.df.to_csv(self.fname, float_format=lambda x: f"{x:.6f}") + + +class ToCSVMultiIndexUnusedLevels(BaseIO): + fname = "__test__.csv" + + def setup(self): + df = DataFrame({"a": np.random.randn(100_000), "b": 1, "c": 1}) + self.df = df.set_index(["a", "b"]) + self.df_unused_levels = self.df.iloc[:10_000] + self.df_single_index = df.set_index(["a"]).iloc[:10_000] + + def time_full_frame(self): + self.df.to_csv(self.fname) + + def time_sliced_frame(self): + self.df_unused_levels.to_csv(self.fname) + + def time_single_index_frame(self): + self.df_single_index.to_csv(self.fname) + + +class ToCSVDatetime(BaseIO): + fname = "__test__.csv" + + def setup(self): + rng = date_range("1/1/2000", periods=1000) + self.data = DataFrame(rng, index=rng) + + def time_frame_date_formatting(self): + self.data.to_csv(self.fname, date_format="%Y%m%d") + + +class ToCSVDatetimeIndex(BaseIO): + fname = "__test__.csv" + + def setup(self): + rng = date_range("2000", periods=100_000, freq="s") + self.data = DataFrame({"a": 1}, index=rng) + + def time_frame_date_formatting_index(self): + self.data.to_csv(self.fname, date_format="%Y-%m-%d %H:%M:%S") + + def time_frame_date_no_format_index(self): + self.data.to_csv(self.fname) + + +class ToCSVPeriod(BaseIO): + fname = "__test__.csv" + + params = ([1000, 10000], ["D", "h"]) + param_names = ["nobs", "freq"] + + def setup(self, nobs, freq): + rng = period_range(start="2000-01-01", periods=nobs, freq=freq) + self.data = DataFrame(rng) + if freq == "D": + self.default_fmt = "%Y-%m-%d" + elif freq == "h": + self.default_fmt = "%Y-%m-%d %H:00" + + def time_frame_period_formatting_default(self, nobs, freq): + self.data.to_csv(self.fname) + + def time_frame_period_formatting_default_explicit(self, nobs, freq): + self.data.to_csv(self.fname, date_format=self.default_fmt) + + def time_frame_period_formatting(self, nobs, freq): + # Nb: `date_format` is not actually taken into account here today, so the + # performance is currently identical to `time_frame_period_formatting_default` + # above. This timer is therefore expected to degrade when GH#51621 is fixed. + # (Remove this comment when GH#51621 is fixed.) + self.data.to_csv(self.fname, date_format="%Y-%m-%d___%H:%M:%S") + + +class ToCSVPeriodIndex(BaseIO): + fname = "__test__.csv" + + params = ([1000, 10000], ["D", "h"]) + param_names = ["nobs", "freq"] + + def setup(self, nobs, freq): + rng = period_range(start="2000-01-01", periods=nobs, freq=freq) + self.data = DataFrame({"a": 1}, index=rng) + if freq == "D": + self.default_fmt = "%Y-%m-%d" + elif freq == "h": + self.default_fmt = "%Y-%m-%d %H:00" + + def time_frame_period_formatting_index(self, nobs, freq): + self.data.to_csv(self.fname, date_format="%Y-%m-%d___%H:%M:%S") + + def time_frame_period_formatting_index_default(self, nobs, freq): + self.data.to_csv(self.fname) + + def time_frame_period_formatting_index_default_explicit(self, nobs, freq): + self.data.to_csv(self.fname, date_format=self.default_fmt) + + +class ToCSVDatetimeBig(BaseIO): + fname = "__test__.csv" + timeout = 1500 + params = [1000, 10000, 100000] + param_names = ["nobs"] + + def setup(self, nobs): + d = "2018-11-29" + dt = "2018-11-26 11:18:27.0" + self.data = DataFrame( + { + "dt": [np.datetime64(dt)] * nobs, + "d": [np.datetime64(d)] * nobs, + "r": [np.random.uniform()] * nobs, + } + ) + + def time_frame(self, nobs): + self.data.to_csv(self.fname) + + +class ToCSVIndexes(BaseIO): + fname = "__test__.csv" + + @staticmethod + def _create_df(rows, cols): + index_cols = { + "index1": np.random.randint(0, rows, rows), + "index2": np.full(rows, 1, dtype=int), + "index3": np.full(rows, 1, dtype=int), + } + data_cols = { + f"col{i}": np.random.uniform(0, 100000.0, rows) for i in range(cols) + } + df = DataFrame({**index_cols, **data_cols}) + return df + + def setup(self): + ROWS = 100000 + COLS = 5 + # For tests using .head(), create an initial dataframe with this many times + # more rows + HEAD_ROW_MULTIPLIER = 10 + + self.df_standard_index = self._create_df(ROWS, COLS) + + self.df_custom_index_then_head = ( + self._create_df(ROWS * HEAD_ROW_MULTIPLIER, COLS) + .set_index(["index1", "index2", "index3"]) + .head(ROWS) + ) + + self.df_head_then_custom_index = ( + self._create_df(ROWS * HEAD_ROW_MULTIPLIER, COLS) + .head(ROWS) + .set_index(["index1", "index2", "index3"]) + ) + + def time_standard_index(self): + self.df_standard_index.to_csv(self.fname) + + def time_multiindex(self): + self.df_head_then_custom_index.to_csv(self.fname) + + def time_head_of_multiindex(self): + self.df_custom_index_then_head.to_csv(self.fname) + + +class StringIORewind: + def data(self, stringio_object): + stringio_object.seek(0) + return stringio_object + + +class ReadCSVDInferDatetimeFormat(StringIORewind): + params = [None, "custom", "iso8601", "ymd"] + param_names = ["format"] + + def setup(self, format): + rng = date_range("1/1/2000", periods=1000) + formats = { + None: None, + "custom": "%m/%d/%Y %H:%M:%S.%f", + "iso8601": "%Y-%m-%d %H:%M:%S", + "ymd": "%Y%m%d", + } + dt_format = formats[format] + self.StringIO_input = StringIO("\n".join(rng.strftime(dt_format).tolist())) + + def time_read_csv(self, format): + read_csv( + self.data(self.StringIO_input), + header=None, + names=["foo"], + parse_dates=["foo"], + ) + + +class ReadCSVConcatDatetime(StringIORewind): + iso8601 = "%Y-%m-%d %H:%M:%S" + + def setup(self): + rng = date_range("1/1/2000", periods=50000, freq="s") + self.StringIO_input = StringIO("\n".join(rng.strftime(self.iso8601).tolist())) + + def time_read_csv(self): + read_csv( + self.data(self.StringIO_input), + header=None, + names=["foo"], + parse_dates=["foo"], + ) + + +class ReadCSVConcatDatetimeBadDateValue(StringIORewind): + params = (["nan", "0", ""],) + param_names = ["bad_date_value"] + + def setup(self, bad_date_value): + self.StringIO_input = StringIO((f"{bad_date_value},\n") * 50000) + + def time_read_csv(self, bad_date_value): + read_csv( + self.data(self.StringIO_input), + header=None, + names=["foo", "bar"], + parse_dates=["foo"], + ) + + +class ReadCSVSkipRows(BaseIO): + fname = "__test__.csv" + params = ([None, 10000], ["c", "python", "pyarrow"]) + param_names = ["skiprows", "engine"] + + def setup(self, skiprows, engine): + N = 20000 + index = Index([f"i-{i}" for i in range(N)], dtype=object) + df = DataFrame( + { + "float1": np.random.randn(N), + "float2": np.random.randn(N), + "string1": ["foo"] * N, + "bool1": [True] * N, + "int1": np.random.randint(0, N, size=N), + }, + index=index, + ) + df.to_csv(self.fname) + + def time_skipprows(self, skiprows, engine): + read_csv(self.fname, skiprows=skiprows, engine=engine) + + +class ReadUint64Integers(StringIORewind): + def setup(self): + self.na_values = [2**63 + 500] + arr = np.arange(10000).astype("uint64") + 2**63 + self.data1 = StringIO("\n".join(arr.astype(str).tolist())) + arr = arr.astype(object) + arr[500] = -1 + self.data2 = StringIO("\n".join(arr.astype(str).tolist())) + + def time_read_uint64(self): + read_csv(self.data(self.data1), header=None, names=["foo"]) + + def time_read_uint64_neg_values(self): + read_csv(self.data(self.data2), header=None, names=["foo"]) + + def time_read_uint64_na_values(self): + read_csv( + self.data(self.data1), header=None, names=["foo"], na_values=self.na_values + ) + + +class ReadCSVThousands(BaseIO): + fname = "__test__.csv" + params = ([",", "|"], [None, ","], ["c", "python"]) + param_names = ["sep", "thousands", "engine"] + + def setup(self, sep, thousands, engine): + N = 10000 + K = 8 + data = np.random.randn(N, K) * np.random.randint(100, 10000, (N, K)) + df = DataFrame(data) + if thousands is not None: + fmt = f":{thousands}" + fmt = "{" + fmt + "}" + df = df.map(lambda x: fmt.format(x)) + df.to_csv(self.fname, sep=sep) + + def time_thousands(self, sep, thousands, engine): + read_csv(self.fname, sep=sep, thousands=thousands, engine=engine) + + +class ReadCSVComment(StringIORewind): + params = ["c", "python"] + param_names = ["engine"] + + def setup(self, engine): + data = ["A,B,C"] + (["1,2,3 # comment"] * 100000) + self.StringIO_input = StringIO("\n".join(data)) + + def time_comment(self, engine): + read_csv( + self.data(self.StringIO_input), comment="#", header=None, names=list("abc") + ) + + +class ReadCSVFloatPrecision(StringIORewind): + params = ([",", ";"], [".", "_"], [None, "high", "round_trip"]) + param_names = ["sep", "decimal", "float_precision"] + + def setup(self, sep, decimal, float_precision): + floats = [ + "".join([random.choice(string.digits) for _ in range(28)]) + for _ in range(15) + ] + rows = sep.join([f"0{decimal}{{}}"] * 3) + "\n" + data = rows * 5 + data = data.format(*floats) * 200 # 1000 x 3 strings csv + self.StringIO_input = StringIO(data) + + def time_read_csv(self, sep, decimal, float_precision): + read_csv( + self.data(self.StringIO_input), + sep=sep, + header=None, + names=list("abc"), + float_precision=float_precision, + ) + + def time_read_csv_python_engine(self, sep, decimal, float_precision): + read_csv( + self.data(self.StringIO_input), + sep=sep, + header=None, + engine="python", + float_precision=None, + names=list("abc"), + ) + + +class ReadCSVEngine(StringIORewind): + params = ["c", "python", "pyarrow"] + param_names = ["engine"] + + def setup(self, engine): + data = ["A,B,C,D,E"] + (["1,2,3,4,5"] * 100000) + self.StringIO_input = StringIO("\n".join(data)) + # simulate reading from file + self.BytesIO_input = BytesIO(self.StringIO_input.read().encode("utf-8")) + + def time_read_stringcsv(self, engine): + read_csv(self.data(self.StringIO_input), engine=engine) + + def time_read_bytescsv(self, engine): + read_csv(self.data(self.BytesIO_input), engine=engine) + + def peakmem_read_csv(self, engine): + read_csv(self.data(self.BytesIO_input), engine=engine) + + +class ReadCSVCategorical(BaseIO): + fname = "__test__.csv" + params = ["c", "python"] + param_names = ["engine"] + + def setup(self, engine): + N = 100000 + group1 = ["aaaaaaaa", "bbbbbbb", "cccccccc", "dddddddd", "eeeeeeee"] + df = DataFrame(np.random.choice(group1, (N, 3)), columns=list("abc")) + df.to_csv(self.fname, index=False) + + def time_convert_post(self, engine): + read_csv(self.fname, engine=engine).apply(Categorical) + + def time_convert_direct(self, engine): + read_csv(self.fname, engine=engine, dtype="category") + + +class ReadCSVParseDates(StringIORewind): + params = ["c", "python"] + param_names = ["engine"] + + def setup(self, engine): + data = """{},19:00:00,18:56:00,0.8100,2.8100,7.2000,0.0000,280.0000\n + {},20:00:00,19:56:00,0.0100,2.2100,7.2000,0.0000,260.0000\n + {},21:00:00,20:56:00,-0.5900,2.2100,5.7000,0.0000,280.0000\n + {},21:00:00,21:18:00,-0.9900,2.0100,3.6000,0.0000,270.0000\n + {},22:00:00,21:56:00,-0.5900,1.7100,5.1000,0.0000,290.0000\n + """ + two_cols = ["KORD,19990127"] * 5 + data = data.format(*two_cols) + self.StringIO_input = StringIO(data) + + def time_baseline(self, engine): + read_csv( + self.data(self.StringIO_input), + engine=engine, + sep=",", + header=None, + parse_dates=[1], + names=list(string.digits[:9]), + ) + + +class ReadCSVCachedParseDates(StringIORewind): + params = ([True, False], ["c", "python"]) + param_names = ["do_cache", "engine"] + + def setup(self, do_cache, engine): + data = ("\n".join([f"10/{year}" for year in range(2000, 2100)]) + "\n") * 10 + self.StringIO_input = StringIO(data) + + def time_read_csv_cached(self, do_cache, engine): + try: + read_csv( + self.data(self.StringIO_input), + engine=engine, + header=None, + parse_dates=[0], + cache_dates=do_cache, + ) + except TypeError: + # cache_dates is a new keyword in 0.25 + pass + + +class ReadCSVMemoryGrowth(BaseIO): + chunksize = 20 + num_rows = 1000 + fname = "__test__.csv" + params = ["c", "python"] + param_names = ["engine"] + + def setup(self, engine): + with open(self.fname, "w", encoding="utf-8") as f: + for i in range(self.num_rows): + f.write(f"{i}\n") + + def mem_parser_chunks(self, engine): + # see gh-24805. + result = read_csv(self.fname, chunksize=self.chunksize, engine=engine) + + for _ in result: + pass + + +class ReadCSVParseSpecialDate(StringIORewind): + params = (["mY", "mdY", "hm"], ["c", "python"]) + param_names = ["value", "engine"] + objects = { + "mY": "01-2019\n10-2019\n02/2000\n", + "mdY": "12/02/2010\n", + "hm": "21:34\n", + } + + def setup(self, value, engine): + count_elem = 10000 + data = self.objects[value] * count_elem + self.StringIO_input = StringIO(data) + + def time_read_special_date(self, value, engine): + read_csv( + self.data(self.StringIO_input), + engine=engine, + sep=",", + header=None, + names=["Date"], + parse_dates=["Date"], + ) + + +class ReadCSVMemMapUTF8: + fname = "__test__.csv" + number = 5 + + def setup(self): + lines = [] + line_length = 128 + start_char = " " + end_char = "\U00010080" + # This for loop creates a list of 128-char strings + # consisting of consecutive Unicode chars + for lnum in range(ord(start_char), ord(end_char), line_length): + line = "".join([chr(c) for c in range(lnum, lnum + 0x80)]) + "\n" + try: + line.encode("utf-8") + except UnicodeEncodeError: + # Some 16-bit words are not valid Unicode chars and must be skipped + continue + lines.append(line) + df = DataFrame(lines) + df = concat([df for n in range(100)], ignore_index=True) + df.to_csv(self.fname, index=False, header=False, encoding="utf-8") + + def time_read_memmapped_utf8(self): + read_csv(self.fname, header=None, memory_map=True, encoding="utf-8", engine="c") + + +class ParseDateComparison(StringIORewind): + params = ([False, True],) + param_names = ["cache_dates"] + + def setup(self, cache_dates): + count_elem = 10000 + data = "12-02-2010\n" * count_elem + self.StringIO_input = StringIO(data) + + def time_read_csv_dayfirst(self, cache_dates): + try: + read_csv( + self.data(self.StringIO_input), + sep=",", + header=None, + names=["Date"], + parse_dates=["Date"], + cache_dates=cache_dates, + dayfirst=True, + ) + except TypeError: + # cache_dates is a new keyword in 0.25 + pass + + def time_to_datetime_dayfirst(self, cache_dates): + df = read_csv( + self.data(self.StringIO_input), dtype={"date": str}, names=["date"] + ) + to_datetime(df["date"], cache=cache_dates, dayfirst=True) + + def time_to_datetime_format_DD_MM_YYYY(self, cache_dates): + df = read_csv( + self.data(self.StringIO_input), dtype={"date": str}, names=["date"] + ) + to_datetime(df["date"], cache=cache_dates, format="%d-%m-%Y") + + +class ReadCSVIndexCol(StringIORewind): + def setup(self): + count_elem = 100_000 + data = "a,b\n" + "1,2\n" * count_elem + self.StringIO_input = StringIO(data) + + def time_read_csv_index_col(self): + read_csv(self.data(self.StringIO_input), index_col="a") + + +class ReadCSVDatePyarrowEngine(StringIORewind): + def setup(self): + count_elem = 100_000 + data = "a\n" + "2019-12-31\n" * count_elem + self.StringIO_input = StringIO(data) + + def time_read_csv_index_col(self): + read_csv( + self.data(self.StringIO_input), + parse_dates=["a"], + engine="pyarrow", + dtype_backend="pyarrow", + ) + + +class ReadCSVCParserLowMemory: + # GH 16798 + def setup(self): + self.csv = StringIO( + "strings\n" + "\n".join(["x" * (1 << 20) for _ in range(2100)]) + ) + + def peakmem_over_2gb_input(self): + read_csv(self.csv, engine="c", low_memory=False) + + +from ..pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/io/excel.py b/asv_bench/benchmarks/io/excel.py new file mode 100644 index 0000000000000..902a61be901bd --- /dev/null +++ b/asv_bench/benchmarks/io/excel.py @@ -0,0 +1,108 @@ +from io import BytesIO + +import numpy as np +from odf.opendocument import OpenDocumentSpreadsheet +from odf.table import ( + Table, + TableCell, + TableRow, +) +from odf.text import P + +from pandas import ( + DataFrame, + ExcelWriter, + Index, + date_range, + read_excel, +) + + +def _generate_dataframe(): + N = 2000 + C = 5 + df = DataFrame( + np.random.randn(N, C), + columns=[f"float{i}" for i in range(C)], + index=date_range("20000101", periods=N, freq="h"), + ) + df["object"] = Index([f"i-{i}" for i in range(N)], dtype=object) + return df + + +class WriteExcel: + params = ["openpyxl", "xlsxwriter"] + param_names = ["engine"] + + def setup(self, engine): + self.df = _generate_dataframe() + + def time_write_excel(self, engine): + bio = BytesIO() + bio.seek(0) + with ExcelWriter(bio, engine=engine) as writer: + self.df.to_excel(writer, sheet_name="Sheet1") + + +class WriteExcelStyled: + params = ["openpyxl", "xlsxwriter"] + param_names = ["engine"] + + def setup(self, engine): + self.df = _generate_dataframe() + + def time_write_excel_style(self, engine): + bio = BytesIO() + bio.seek(0) + with ExcelWriter(bio, engine=engine) as writer: + df_style = self.df.style + df_style.map(lambda x: "border: red 1px solid;") + df_style.map(lambda x: "color: blue") + df_style.map(lambda x: "border-color: green black", subset=["float1"]) + df_style.to_excel(writer, sheet_name="Sheet1") + + +class ReadExcel: + params = ["openpyxl", "odf"] + param_names = ["engine"] + fname_excel = "spreadsheet.xlsx" + fname_odf = "spreadsheet.ods" + + def _create_odf(self): + doc = OpenDocumentSpreadsheet() + table = Table(name="Table1") + for row in self.df.values: + tr = TableRow() + for val in row: + tc = TableCell(valuetype="string") + tc.addElement(P(text=val)) + tr.addElement(tc) + table.addElement(tr) + + doc.spreadsheet.addElement(table) + doc.save(self.fname_odf) + + def setup_cache(self): + self.df = _generate_dataframe() + + self.df.to_excel(self.fname_excel, sheet_name="Sheet1") + self._create_odf() + + def time_read_excel(self, engine): + if engine == "odf": + fname = self.fname_odf + else: + fname = self.fname_excel + read_excel(fname, engine=engine) + + +class ReadExcelNRows(ReadExcel): + def time_read_excel(self, engine): + if engine == "odf": + fname = self.fname_odf + else: + fname = self.fname_excel + read_excel(fname, engine=engine, nrows=10) + + +from ..pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/io/hdf.py b/asv_bench/benchmarks/io/hdf.py new file mode 100644 index 0000000000000..2eb4c8c7f674b --- /dev/null +++ b/asv_bench/benchmarks/io/hdf.py @@ -0,0 +1,143 @@ +import numpy as np + +from pandas import ( + DataFrame, + HDFStore, + Index, + date_range, + read_hdf, +) + +from ..pandas_vb_common import BaseIO + + +class HDFStoreDataFrame(BaseIO): + def setup(self): + N = 25000 + index = Index([f"i-{i}" for i in range(N)], dtype=object) + self.df = DataFrame( + {"float1": np.random.randn(N), "float2": np.random.randn(N)}, index=index + ) + self.df_mixed = DataFrame( + { + "float1": np.random.randn(N), + "float2": np.random.randn(N), + "string1": ["foo"] * N, + "bool1": [True] * N, + "int1": np.random.randint(0, N, size=N), + }, + index=index, + ) + self.df_wide = DataFrame(np.random.randn(N, 100)) + self.start_wide = self.df_wide.index[10000] + self.stop_wide = self.df_wide.index[15000] + self.df2 = DataFrame( + {"float1": np.random.randn(N), "float2": np.random.randn(N)}, + index=date_range("1/1/2000", periods=N), + ) + self.start = self.df2.index[10000] + self.stop = self.df2.index[15000] + self.df_wide2 = DataFrame( + np.random.randn(N, 100), index=date_range("1/1/2000", periods=N) + ) + self.df_dc = DataFrame( + np.random.randn(N, 10), columns=[f"C{i:03d}" for i in range(10)] + ) + + self.fname = "__test__.h5" + + self.store = HDFStore(self.fname) + self.store.put("fixed", self.df) + self.store.put("fixed_mixed", self.df_mixed) + self.store.append("table", self.df2) + self.store.append("table_mixed", self.df_mixed) + self.store.append("table_wide", self.df_wide) + self.store.append("table_wide2", self.df_wide2) + + def teardown(self): + self.store.close() + self.remove(self.fname) + + def time_read_store(self): + self.store.get("fixed") + + def time_read_store_mixed(self): + self.store.get("fixed_mixed") + + def time_write_store(self): + self.store.put("fixed_write", self.df) + + def time_write_store_mixed(self): + self.store.put("fixed_mixed_write", self.df_mixed) + + def time_read_store_table_mixed(self): + self.store.select("table_mixed") + + def time_write_store_table_mixed(self): + self.store.append("table_mixed_write", self.df_mixed) + + def time_read_store_table(self): + self.store.select("table") + + def time_write_store_table(self): + self.store.append("table_write", self.df) + + def time_read_store_table_wide(self): + self.store.select("table_wide") + + def time_write_store_table_wide(self): + self.store.append("table_wide_write", self.df_wide) + + def time_write_store_table_dc(self): + self.store.append("table_dc_write", self.df_dc, data_columns=True) + + def time_query_store_table_wide(self): + self.store.select( + "table_wide", where="index > self.start_wide and index < self.stop_wide" + ) + + def time_query_store_table(self): + self.store.select("table", where="index > self.start and index < self.stop") + + def time_store_repr(self): + repr(self.store) + + def time_store_str(self): + str(self.store) + + def time_store_info(self): + self.store.info() + + +class HDF(BaseIO): + params = ["table", "fixed"] + param_names = ["format"] + + def setup(self, format): + self.fname = "__test__.h5" + N = 100000 + C = 5 + self.df = DataFrame( + np.random.randn(N, C), + columns=[f"float{i}" for i in range(C)], + index=date_range("20000101", periods=N, freq="h"), + ) + self.df["object"] = Index([f"i-{i}" for i in range(N)], dtype=object) + self.df.to_hdf(self.fname, key="df", format=format) + + # Numeric df + self.df1 = self.df.copy() + self.df1 = self.df1.reset_index() + self.df1.to_hdf(self.fname, key="df1", format=format) + + def time_read_hdf(self, format): + read_hdf(self.fname, "df") + + def peakmem_read_hdf(self, format): + read_hdf(self.fname, "df") + + def time_write_hdf(self, format): + self.df.to_hdf(self.fname, key="df", format=format) + + +from ..pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/io/json.py b/asv_bench/benchmarks/io/json.py new file mode 100644 index 0000000000000..bcbfcdea42dd9 --- /dev/null +++ b/asv_bench/benchmarks/io/json.py @@ -0,0 +1,312 @@ +import sys + +import numpy as np + +from pandas import ( + DataFrame, + Index, + concat, + date_range, + json_normalize, + read_json, + timedelta_range, +) + +from ..pandas_vb_common import BaseIO + + +class ReadJSON(BaseIO): + fname = "__test__.json" + params = (["split", "index", "records"], ["int", "datetime"]) + param_names = ["orient", "index"] + + def setup(self, orient, index): + N = 100000 + indexes = { + "int": np.arange(N), + "datetime": date_range("20000101", periods=N, freq="h"), + } + df = DataFrame( + np.random.randn(N, 5), + columns=[f"float_{i}" for i in range(5)], + index=indexes[index], + ) + df.to_json(self.fname, orient=orient) + + def time_read_json(self, orient, index): + read_json(self.fname, orient=orient) + + +class ReadJSONLines(BaseIO): + fname = "__test_lines__.json" + params = ["int", "datetime"] + param_names = ["index"] + + def setup(self, index): + N = 100000 + indexes = { + "int": np.arange(N), + "datetime": date_range("20000101", periods=N, freq="h"), + } + df = DataFrame( + np.random.randn(N, 5), + columns=[f"float_{i}" for i in range(5)], + index=indexes[index], + ) + df.to_json(self.fname, orient="records", lines=True) + + def time_read_json_lines(self, index): + read_json(self.fname, orient="records", lines=True) + + def time_read_json_lines_concat(self, index): + concat(read_json(self.fname, orient="records", lines=True, chunksize=25000)) + + def time_read_json_lines_nrows(self, index): + read_json(self.fname, orient="records", lines=True, nrows=25000) + + def peakmem_read_json_lines(self, index): + read_json(self.fname, orient="records", lines=True) + + def peakmem_read_json_lines_concat(self, index): + concat(read_json(self.fname, orient="records", lines=True, chunksize=25000)) + + def peakmem_read_json_lines_nrows(self, index): + read_json(self.fname, orient="records", lines=True, nrows=15000) + + +class NormalizeJSON(BaseIO): + fname = "__test__.json" + params = [ + ["split", "columns", "index", "values", "records"], + ["df", "df_date_idx", "df_td_int_ts", "df_int_floats", "df_int_float_str"], + ] + param_names = ["orient", "frame"] + + def setup(self, orient, frame): + data = { + "hello": ["thisisatest", 999898, "mixed types"], + "nest1": {"nest2": {"nest3": "nest3_value", "nest3_int": 3445}}, + "nest1_list": {"nest2": ["blah", 32423, 546456.876, 92030234]}, + "hello2": "string", + } + self.data = [data for i in range(10000)] + + def time_normalize_json(self, orient, frame): + json_normalize(self.data) + + +class ToJSON(BaseIO): + fname = "__test__.json" + params = [ + ["split", "columns", "index", "values", "records"], + ["df", "df_date_idx", "df_td_int_ts", "df_int_floats", "df_int_float_str"], + ] + param_names = ["orient", "frame"] + + def setup(self, orient, frame): + N = 10**5 + ncols = 5 + index = date_range("20000101", periods=N, freq="h") + timedeltas = timedelta_range(start=1, periods=N, freq="s") + datetimes = date_range(start=1, periods=N, freq="s") + ints = np.random.randint(100000000, size=N) + longints = sys.maxsize * np.random.randint(100000000, size=N) + floats = np.random.randn(N) + strings = Index([f"i-{i}" for i in range(N)], dtype=object) + self.df = DataFrame(np.random.randn(N, ncols), index=np.arange(N)) + self.df_date_idx = DataFrame(np.random.randn(N, ncols), index=index) + self.df_td_int_ts = DataFrame( + { + "td_1": timedeltas, + "td_2": timedeltas, + "int_1": ints, + "int_2": ints, + "ts_1": datetimes, + "ts_2": datetimes, + }, + index=index, + ) + self.df_int_floats = DataFrame( + { + "int_1": ints, + "int_2": ints, + "int_3": ints, + "float_1": floats, + "float_2": floats, + "float_3": floats, + }, + index=index, + ) + self.df_int_float_str = DataFrame( + { + "int_1": ints, + "int_2": ints, + "float_1": floats, + "float_2": floats, + "str_1": strings, + "str_2": strings, + }, + index=index, + ) + + self.df_longint_float_str = DataFrame( + { + "longint_1": longints, + "longint_2": longints, + "float_1": floats, + "float_2": floats, + "str_1": strings, + "str_2": strings, + }, + index=index, + ) + + def time_to_json(self, orient, frame): + getattr(self, frame).to_json(self.fname, orient=orient) + + def peakmem_to_json(self, orient, frame): + getattr(self, frame).to_json(self.fname, orient=orient) + + +class ToJSONWide(ToJSON): + def setup(self, orient, frame): + super().setup(orient, frame) + base_df = getattr(self, frame).copy() + df_wide = concat([base_df.iloc[:100]] * 1000, ignore_index=True, axis=1) + self.df_wide = df_wide + + def time_to_json_wide(self, orient, frame): + self.df_wide.to_json(self.fname, orient=orient) + + def peakmem_to_json_wide(self, orient, frame): + self.df_wide.to_json(self.fname, orient=orient) + + +class ToJSONISO(BaseIO): + fname = "__test__.json" + params = [["split", "columns", "index", "values", "records"]] + param_names = ["orient"] + + def setup(self, orient): + N = 10**5 + index = date_range("20000101", periods=N, freq="h") + timedeltas = timedelta_range(start=1, periods=N, freq="s") + datetimes = date_range(start=1, periods=N, freq="s") + self.df = DataFrame( + { + "td_1": timedeltas, + "td_2": timedeltas, + "ts_1": datetimes, + "ts_2": datetimes, + }, + index=index, + ) + + def time_iso_format(self, orient): + self.df.to_json(orient=orient, date_format="iso") + + +class ToJSONLines(BaseIO): + fname = "__test__.json" + + def setup(self): + N = 10**5 + ncols = 5 + index = date_range("20000101", periods=N, freq="h") + timedeltas = timedelta_range(start=1, periods=N, freq="s") + datetimes = date_range(start=1, periods=N, freq="s") + ints = np.random.randint(100000000, size=N) + longints = sys.maxsize * np.random.randint(100000000, size=N) + floats = np.random.randn(N) + strings = Index([f"i-{i}" for i in range(N)], dtype=object) + self.df = DataFrame(np.random.randn(N, ncols), index=np.arange(N)) + self.df_date_idx = DataFrame(np.random.randn(N, ncols), index=index) + self.df_td_int_ts = DataFrame( + { + "td_1": timedeltas, + "td_2": timedeltas, + "int_1": ints, + "int_2": ints, + "ts_1": datetimes, + "ts_2": datetimes, + }, + index=index, + ) + self.df_int_floats = DataFrame( + { + "int_1": ints, + "int_2": ints, + "int_3": ints, + "float_1": floats, + "float_2": floats, + "float_3": floats, + }, + index=index, + ) + self.df_int_float_str = DataFrame( + { + "int_1": ints, + "int_2": ints, + "float_1": floats, + "float_2": floats, + "str_1": strings, + "str_2": strings, + }, + index=index, + ) + self.df_longint_float_str = DataFrame( + { + "longint_1": longints, + "longint_2": longints, + "float_1": floats, + "float_2": floats, + "str_1": strings, + "str_2": strings, + }, + index=index, + ) + + def time_floats_with_int_idex_lines(self): + self.df.to_json(self.fname, orient="records", lines=True) + + def time_floats_with_dt_index_lines(self): + self.df_date_idx.to_json(self.fname, orient="records", lines=True) + + def time_delta_int_tstamp_lines(self): + self.df_td_int_ts.to_json(self.fname, orient="records", lines=True) + + def time_float_int_lines(self): + self.df_int_floats.to_json(self.fname, orient="records", lines=True) + + def time_float_int_str_lines(self): + self.df_int_float_str.to_json(self.fname, orient="records", lines=True) + + def time_float_longint_str_lines(self): + self.df_longint_float_str.to_json(self.fname, orient="records", lines=True) + + +class ToJSONMem: + def setup_cache(self): + df = DataFrame([[1]]) + df2 = DataFrame(range(8), date_range("1/1/2000", periods=8, freq="min")) + frames = {"int": df, "float": df.astype(float), "datetime": df2} + + return frames + + def peakmem_int(self, frames): + df = frames["int"] + for _ in range(100_000): + df.to_json() + + def peakmem_float(self, frames): + df = frames["float"] + for _ in range(100_000): + df.to_json() + + def peakmem_time(self, frames): + df = frames["datetime"] + for _ in range(10_000): + df.to_json(orient="table") + + +from ..pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/io/parsers.py b/asv_bench/benchmarks/io/parsers.py new file mode 100644 index 0000000000000..d3fd5075a4707 --- /dev/null +++ b/asv_bench/benchmarks/io/parsers.py @@ -0,0 +1,17 @@ +try: + from pandas._libs.tslibs.parsing import _does_string_look_like_datetime +except ImportError: + # Avoid whole benchmark suite import failure on asv (currently 0.4) + pass + + +class DoesStringLookLikeDatetime: + params = (["2Q2005", "0.0", "10000"],) + param_names = ["value"] + + def setup(self, value): + self.objects = [value] * 1000000 + + def time_check_datetimes(self, value): + for obj in self.objects: + _does_string_look_like_datetime(obj) diff --git a/asv_bench/benchmarks/io/pickle.py b/asv_bench/benchmarks/io/pickle.py new file mode 100644 index 0000000000000..4787b57b54756 --- /dev/null +++ b/asv_bench/benchmarks/io/pickle.py @@ -0,0 +1,39 @@ +import numpy as np + +from pandas import ( + DataFrame, + Index, + date_range, + read_pickle, +) + +from ..pandas_vb_common import BaseIO + + +class Pickle(BaseIO): + def setup(self): + self.fname = "__test__.pkl" + N = 100000 + C = 5 + self.df = DataFrame( + np.random.randn(N, C), + columns=[f"float{i}" for i in range(C)], + index=date_range("20000101", periods=N, freq="h"), + ) + self.df["object"] = Index([f"i-{i}" for i in range(N)], dtype=object) + self.df.to_pickle(self.fname) + + def time_read_pickle(self): + read_pickle(self.fname) + + def time_write_pickle(self): + self.df.to_pickle(self.fname) + + def peakmem_read_pickle(self): + read_pickle(self.fname) + + def peakmem_write_pickle(self): + self.df.to_pickle(self.fname) + + +from ..pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/io/sas.py b/asv_bench/benchmarks/io/sas.py new file mode 100644 index 0000000000000..411e5b6099f76 --- /dev/null +++ b/asv_bench/benchmarks/io/sas.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from pandas import read_sas + +ROOT = Path(__file__).parents[3] / "pandas" / "tests" / "io" / "sas" / "data" + + +class SAS: + def time_read_sas7bdat(self): + read_sas(ROOT / "test1.sas7bdat") + + def time_read_xpt(self): + read_sas(ROOT / "paxraw_d_short.xpt") + + def time_read_sas7bdat_2(self): + next(read_sas(ROOT / "0x00controlbyte.sas7bdat.bz2", chunksize=11000)) + + def time_read_sas7bdat_2_chunked(self): + for i, _ in enumerate( + read_sas(ROOT / "0x00controlbyte.sas7bdat.bz2", chunksize=1000) + ): + if i == 10: + break diff --git a/asv_bench/benchmarks/io/sql.py b/asv_bench/benchmarks/io/sql.py new file mode 100644 index 0000000000000..e87cc4aaa80c7 --- /dev/null +++ b/asv_bench/benchmarks/io/sql.py @@ -0,0 +1,173 @@ +import sqlite3 + +import numpy as np +from sqlalchemy import create_engine + +from pandas import ( + DataFrame, + Index, + date_range, + read_sql_query, + read_sql_table, +) + + +class SQL: + params = ["sqlalchemy", "sqlite"] + param_names = ["connection"] + + def setup(self, connection): + N = 10000 + con = { + "sqlalchemy": create_engine("sqlite:///:memory:"), + "sqlite": sqlite3.connect(":memory:"), + } + self.table_name = "test_type" + self.query_all = f"SELECT * FROM {self.table_name}" + self.con = con[connection] + self.df = DataFrame( + { + "float": np.random.randn(N), + "float_with_nan": np.random.randn(N), + "string": ["foo"] * N, + "bool": [True] * N, + "int": np.random.randint(0, N, size=N), + "datetime": date_range("2000-01-01", periods=N, freq="s"), + }, + index=Index([f"i-{i}" for i in range(N)], dtype=object), + ) + self.df.iloc[1000:3000, 1] = np.nan + self.df["date"] = self.df["datetime"].dt.date + self.df["time"] = self.df["datetime"].dt.time + self.df["datetime_string"] = self.df["datetime"].astype(str) + self.df.to_sql(self.table_name, self.con, if_exists="replace") + + def time_to_sql_dataframe(self, connection): + self.df.to_sql("test1", self.con, if_exists="replace") + + def time_read_sql_query(self, connection): + read_sql_query(self.query_all, self.con) + + +class WriteSQLDtypes: + params = ( + ["sqlalchemy", "sqlite"], + [ + "float", + "float_with_nan", + "string", + "bool", + "int", + "date", + "time", + "datetime", + ], + ) + param_names = ["connection", "dtype"] + + def setup(self, connection, dtype): + N = 10000 + con = { + "sqlalchemy": create_engine("sqlite:///:memory:"), + "sqlite": sqlite3.connect(":memory:"), + } + self.table_name = "test_type" + self.query_col = f"SELECT {dtype} FROM {self.table_name}" + self.con = con[connection] + self.df = DataFrame( + { + "float": np.random.randn(N), + "float_with_nan": np.random.randn(N), + "string": ["foo"] * N, + "bool": [True] * N, + "int": np.random.randint(0, N, size=N), + "datetime": date_range("2000-01-01", periods=N, freq="s"), + }, + index=Index([f"i-{i}" for i in range(N)], dtype=object), + ) + self.df.iloc[1000:3000, 1] = np.nan + self.df["date"] = self.df["datetime"].dt.date + self.df["time"] = self.df["datetime"].dt.time + self.df["datetime_string"] = self.df["datetime"].astype(str) + self.df.to_sql(self.table_name, self.con, if_exists="replace") + + def time_to_sql_dataframe_column(self, connection, dtype): + self.df[[dtype]].to_sql("test1", self.con, if_exists="replace") + + def time_read_sql_query_select_column(self, connection, dtype): + read_sql_query(self.query_col, self.con) + + +class ReadSQLTable: + def setup(self): + N = 10000 + self.table_name = "test" + self.con = create_engine("sqlite:///:memory:") + self.df = DataFrame( + { + "float": np.random.randn(N), + "float_with_nan": np.random.randn(N), + "string": ["foo"] * N, + "bool": [True] * N, + "int": np.random.randint(0, N, size=N), + "datetime": date_range("2000-01-01", periods=N, freq="s"), + }, + index=Index([f"i-{i}" for i in range(N)], dtype=object), + ) + self.df.iloc[1000:3000, 1] = np.nan + self.df["date"] = self.df["datetime"].dt.date + self.df["time"] = self.df["datetime"].dt.time + self.df["datetime_string"] = self.df["datetime"].astype(str) + self.df.to_sql(self.table_name, self.con, if_exists="replace") + + def time_read_sql_table_all(self): + read_sql_table(self.table_name, self.con) + + def time_read_sql_table_parse_dates(self): + read_sql_table( + self.table_name, + self.con, + columns=["datetime_string"], + parse_dates=["datetime_string"], + ) + + +class ReadSQLTableDtypes: + params = [ + "float", + "float_with_nan", + "string", + "bool", + "int", + "date", + "time", + "datetime", + ] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10000 + self.table_name = "test" + self.con = create_engine("sqlite:///:memory:") + self.df = DataFrame( + { + "float": np.random.randn(N), + "float_with_nan": np.random.randn(N), + "string": ["foo"] * N, + "bool": [True] * N, + "int": np.random.randint(0, N, size=N), + "datetime": date_range("2000-01-01", periods=N, freq="s"), + }, + index=Index([f"i-{i}" for i in range(N)], dtype=object), + ) + self.df.iloc[1000:3000, 1] = np.nan + self.df["date"] = self.df["datetime"].dt.date + self.df["time"] = self.df["datetime"].dt.time + self.df["datetime_string"] = self.df["datetime"].astype(str) + self.df.to_sql(self.table_name, self.con, if_exists="replace") + + def time_read_sql_table_column(self, dtype): + read_sql_table(self.table_name, self.con, columns=[dtype]) + + +from ..pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/io/stata.py b/asv_bench/benchmarks/io/stata.py new file mode 100644 index 0000000000000..ff33ededdfed9 --- /dev/null +++ b/asv_bench/benchmarks/io/stata.py @@ -0,0 +1,57 @@ +import numpy as np + +from pandas import ( + DataFrame, + Index, + date_range, + read_stata, +) + +from ..pandas_vb_common import BaseIO + + +class Stata(BaseIO): + params = ["tc", "td", "tm", "tw", "th", "tq", "ty"] + param_names = ["convert_dates"] + + def setup(self, convert_dates): + self.fname = "__test__.dta" + N = self.N = 100000 + C = self.C = 5 + self.df = DataFrame( + np.random.randn(N, C), + columns=[f"float{i}" for i in range(C)], + index=date_range("20000101", periods=N, freq="h"), + ) + self.df["object"] = Index([f"i-{i}" for i in range(self.N)], dtype=object) + self.df["int8_"] = np.random.randint( + np.iinfo(np.int8).min, np.iinfo(np.int8).max - 27, N + ) + self.df["int16_"] = np.random.randint( + np.iinfo(np.int16).min, np.iinfo(np.int16).max - 27, N + ) + self.df["int32_"] = np.random.randint( + np.iinfo(np.int32).min, np.iinfo(np.int32).max - 27, N + ) + self.df["float32_"] = np.array(np.random.randn(N), dtype=np.float32) + self.convert_dates = {"index": convert_dates} + self.df.to_stata(self.fname, convert_dates=self.convert_dates) + + def time_read_stata(self, convert_dates): + read_stata(self.fname) + + def time_write_stata(self, convert_dates): + self.df.to_stata(self.fname, convert_dates=self.convert_dates) + + +class StataMissing(Stata): + def setup(self, convert_dates): + super().setup(convert_dates) + for i in range(10): + missing_data = np.random.randn(self.N) + missing_data[missing_data < 0] = np.nan + self.df[f"missing_{i}"] = missing_data + self.df.to_stata(self.fname, convert_dates=self.convert_dates) + + +from ..pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/io/style.py b/asv_bench/benchmarks/io/style.py new file mode 100644 index 0000000000000..0486cabb29845 --- /dev/null +++ b/asv_bench/benchmarks/io/style.py @@ -0,0 +1,93 @@ +import numpy as np + +from pandas import ( + DataFrame, + IndexSlice, +) + + +class Render: + params = [[12, 24, 36], [12, 120]] + param_names = ["cols", "rows"] + + def setup(self, cols, rows): + self.df = DataFrame( + np.random.randn(rows, cols), + columns=[f"float_{i + 1}" for i in range(cols)], + index=[f"row_{i + 1}" for i in range(rows)], + ) + + def time_apply_render(self, cols, rows): + self._style_apply() + self.st._render_html(True, True) + + def peakmem_apply_render(self, cols, rows): + self._style_apply() + self.st._render_html(True, True) + + def time_classes_render(self, cols, rows): + self._style_classes() + self.st._render_html(True, True) + + def peakmem_classes_render(self, cols, rows): + self._style_classes() + self.st._render_html(True, True) + + def time_tooltips_render(self, cols, rows): + self._style_tooltips() + self.st._render_html(True, True) + + def peakmem_tooltips_render(self, cols, rows): + self._style_tooltips() + self.st._render_html(True, True) + + def time_format_render(self, cols, rows): + self._style_format() + self.st._render_html(True, True) + + def peakmem_format_render(self, cols, rows): + self._style_format() + self.st._render_html(True, True) + + def time_apply_format_hide_render(self, cols, rows): + self._style_apply_format_hide() + self.st._render_html(True, True) + + def peakmem_apply_format_hide_render(self, cols, rows): + self._style_apply_format_hide() + self.st._render_html(True, True) + + def _style_apply(self): + def _apply_func(s): + return [ + "background-color: lightcyan" if s.name == "row_1" else "" for v in s + ] + + self.st = self.df.style.apply(_apply_func, axis=1) + + def _style_classes(self): + classes = self.df.map(lambda v: ("cls-1" if v > 0 else "")) + classes.index, classes.columns = self.df.index, self.df.columns + self.st = self.df.style.set_td_classes(classes) + + def _style_format(self): + ic = int(len(self.df.columns) / 4 * 3) + ir = int(len(self.df.index) / 4 * 3) + # apply a formatting function + # subset is flexible but hinders vectorised solutions + self.st = self.df.style.format( + "{:,.3f}", + subset=IndexSlice["row_1" : f"row_{ir}", "float_1" : f"float_{ic}"], + ) + + def _style_apply_format_hide(self): + self.st = self.df.style.map(lambda v: "color: red;") + self.st.format("{:.3f}") + self.st.hide(self.st.index[1:], axis=0) + self.st.hide(self.st.columns[1:], axis=1) + + def _style_tooltips(self): + ttips = DataFrame("abc", index=self.df.index[::2], columns=self.df.columns[::2]) + self.st = self.df.style.set_tooltips(ttips) + self.st.hide(self.st.index[12:], axis=0) + self.st.hide(self.st.columns[12:], axis=1) diff --git a/asv_bench/benchmarks/join_merge.py b/asv_bench/benchmarks/join_merge.py new file mode 100644 index 0000000000000..a6c6990892d38 --- /dev/null +++ b/asv_bench/benchmarks/join_merge.py @@ -0,0 +1,590 @@ +import string + +import numpy as np + +from pandas import ( + DataFrame, + Index, + MultiIndex, + Series, + array, + concat, + date_range, + merge, + merge_asof, +) + +try: + from pandas import merge_ordered +except ImportError: + from pandas import ordered_merge as merge_ordered + + +class Concat: + params = [0, 1] + param_names = ["axis"] + + def setup(self, axis): + N = 1000 + s = Series(N, index=Index([f"i-{i}" for i in range(N)], dtype=object)) + self.series = [s[i:-i] for i in range(1, 10)] * 50 + self.small_frames = [DataFrame(np.random.randn(5, 4))] * 1000 + df = DataFrame( + {"A": range(N)}, index=date_range("20130101", periods=N, freq="s") + ) + self.empty_left = [DataFrame(), df] + self.empty_right = [df, DataFrame()] + self.mixed_ndims = [df, df.head(N // 2)] + + def time_concat_series(self, axis): + concat(self.series, axis=axis, sort=False) + + def time_concat_small_frames(self, axis): + concat(self.small_frames, axis=axis) + + def time_concat_empty_right(self, axis): + concat(self.empty_right, axis=axis) + + def time_concat_empty_left(self, axis): + concat(self.empty_left, axis=axis) + + def time_concat_mixed_ndims(self, axis): + concat(self.mixed_ndims, axis=axis) + + +class ConcatDataFrames: + params = ([0, 1], [True, False]) + param_names = ["axis", "ignore_index"] + + def setup(self, axis, ignore_index): + frame_c = DataFrame(np.zeros((10000, 200), dtype=np.float32, order="C")) + self.frame_c = [frame_c] * 20 + frame_f = DataFrame(np.zeros((10000, 200), dtype=np.float32, order="F")) + self.frame_f = [frame_f] * 20 + + def time_c_ordered(self, axis, ignore_index): + concat(self.frame_c, axis=axis, ignore_index=ignore_index) + + def time_f_ordered(self, axis, ignore_index): + concat(self.frame_f, axis=axis, ignore_index=ignore_index) + + +class ConcatIndexDtype: + params = ( + [ + "datetime64[ns]", + "int64", + "Int64", + "int64[pyarrow]", + "string[python]", + "string[pyarrow]", + ], + ["monotonic", "non_monotonic", "has_na"], + [0, 1], + [True, False], + ) + param_names = ["dtype", "structure", "axis", "sort"] + + def setup(self, dtype, structure, axis, sort): + N = 10_000 + if dtype == "datetime64[ns]": + vals = date_range("1970-01-01", periods=N) + elif dtype in ("int64", "Int64", "int64[pyarrow]"): + vals = np.arange(N, dtype=np.int64) + elif dtype in ("string[python]", "string[pyarrow]"): + vals = Index([f"i-{i}" for i in range(N)], dtype=object) + else: + raise NotImplementedError + + idx = Index(vals, dtype=dtype) + + if structure == "monotonic": + idx = idx.sort_values() + elif structure == "non_monotonic": + idx = idx[::-1] + elif structure == "has_na": + if not idx._can_hold_na: + raise NotImplementedError + idx = Index([None], dtype=dtype).append(idx) + else: + raise NotImplementedError + + self.series = [Series(i, idx[:-i]) for i in range(1, 6)] + + def time_concat_series(self, dtype, structure, axis, sort): + concat(self.series, axis=axis, sort=sort) + + +class Join: + params = [True, False] + param_names = ["sort"] + + def setup(self, sort): + level1 = Index([f"i-{i}" for i in range(10)], dtype=object).values + level2 = Index([f"i-{i}" for i in range(1000)], dtype=object).values + codes1 = np.arange(10).repeat(1000) + codes2 = np.tile(np.arange(1000), 10) + index2 = MultiIndex(levels=[level1, level2], codes=[codes1, codes2]) + self.df_multi = DataFrame( + np.random.randn(len(index2), 4), index=index2, columns=["A", "B", "C", "D"] + ) + + self.key1 = np.tile(level1.take(codes1), 10) + self.key2 = np.tile(level2.take(codes2), 10) + self.df = DataFrame( + { + "data1": np.random.randn(100000), + "data2": np.random.randn(100000), + "key1": self.key1, + "key2": self.key2, + } + ) + + self.df_key1 = DataFrame( + np.random.randn(len(level1), 4), index=level1, columns=["A", "B", "C", "D"] + ) + self.df_key2 = DataFrame( + np.random.randn(len(level2), 4), index=level2, columns=["A", "B", "C", "D"] + ) + + shuf = np.arange(100000) + np.random.shuffle(shuf) + self.df_shuf = self.df.reindex(self.df.index[shuf]) + + def time_join_dataframe_index_multi(self, sort): + self.df.join(self.df_multi, on=["key1", "key2"], sort=sort) + + def time_join_dataframe_index_single_key_bigger(self, sort): + self.df.join(self.df_key2, on="key2", sort=sort) + + def time_join_dataframe_index_single_key_small(self, sort): + self.df.join(self.df_key1, on="key1", sort=sort) + + def time_join_dataframe_index_shuffle_key_bigger_sort(self, sort): + self.df_shuf.join(self.df_key2, on="key2", sort=sort) + + def time_join_dataframes_cross(self, sort): + self.df.loc[:2000].join(self.df_key1, how="cross", sort=sort) + + +class JoinIndex: + def setup(self): + N = 5000 + self.left = DataFrame( + np.random.randint(1, N / 50, (N, 2)), columns=["jim", "joe"] + ) + self.right = DataFrame( + np.random.randint(1, N / 50, (N, 2)), columns=["jolie", "jolia"] + ).set_index("jolie") + + def time_left_outer_join_index(self): + self.left.join(self.right, on="jim") + + +class JoinMultiindexSubset: + def setup(self): + N = 100_000 + mi1 = MultiIndex.from_arrays([np.arange(N)] * 4, names=["a", "b", "c", "d"]) + mi2 = MultiIndex.from_arrays([np.arange(N)] * 2, names=["a", "b"]) + self.left = DataFrame({"col1": 1}, index=mi1) + self.right = DataFrame({"col2": 2}, index=mi2) + + def time_join_multiindex_subset(self): + self.left.join(self.right) + + +class JoinEmpty: + def setup(self): + N = 100_000 + self.df = DataFrame({"A": np.arange(N)}) + self.df_empty = DataFrame(columns=["B", "C"], dtype="int64") + + def time_inner_join_left_empty(self): + self.df_empty.join(self.df, how="inner") + + def time_inner_join_right_empty(self): + self.df.join(self.df_empty, how="inner") + + +class JoinNonUnique: + # outer join of non-unique + # GH 6329 + def setup(self): + date_index = date_range("01-Jan-2013", "23-Jan-2013", freq="min") + daily_dates = date_index.to_period("D").to_timestamp("s", "s") + self.fracofday = date_index.values - daily_dates.values + self.fracofday = self.fracofday.astype("timedelta64[ns]") + self.fracofday = self.fracofday.astype(np.float64) / 86_400_000_000_000 + self.fracofday = Series(self.fracofday, daily_dates) + index = date_range(date_index.min(), date_index.max(), freq="D") + self.temp = Series(1.0, index)[self.fracofday.index] + + def time_join_non_unique_equal(self): + self.fracofday * self.temp + + +class Merge: + params = [True, False] + param_names = ["sort"] + + def setup(self, sort): + N = 10000 + indices = Index([f"i-{i}" for i in range(N)], dtype=object).values + indices2 = Index([f"i-{i}" for i in range(N)], dtype=object).values + key = np.tile(indices[:8000], 10) + key2 = np.tile(indices2[:8000], 10) + self.left = DataFrame( + {"key": key, "key2": key2, "value": np.random.randn(80000)} + ) + self.right = DataFrame( + { + "key": indices[2000:], + "key2": indices2[2000:], + "value2": np.random.randn(8000), + } + ) + + self.df = DataFrame( + { + "key1": np.tile(np.arange(500).repeat(10), 2), + "key2": np.tile(np.arange(250).repeat(10), 4), + "value": np.random.randn(10000), + } + ) + self.df2 = DataFrame({"key1": np.arange(500), "value2": np.random.randn(500)}) + self.df3 = self.df[:5000] + + def time_merge_2intkey(self, sort): + merge(self.left, self.right, sort=sort) + + def time_merge_dataframe_integer_2key(self, sort): + merge(self.df, self.df3, sort=sort) + + def time_merge_dataframe_integer_key(self, sort): + merge(self.df, self.df2, on="key1", sort=sort) + + def time_merge_dataframe_empty_right(self, sort): + merge(self.left, self.right.iloc[:0], sort=sort) + + def time_merge_dataframe_empty_left(self, sort): + merge(self.left.iloc[:0], self.right, sort=sort) + + def time_merge_dataframes_cross(self, sort): + merge(self.left.loc[:2000], self.right.loc[:2000], how="cross", sort=sort) + + +class MergeEA: + params = [ + [ + "Int64", + "Int32", + "Int16", + "UInt64", + "UInt32", + "UInt16", + "Float64", + "Float32", + ], + [True, False], + ] + param_names = ["dtype", "monotonic"] + + def setup(self, dtype, monotonic): + N = 10_000 + indices = np.arange(1, N) + key = np.tile(indices[:8000], 10) + self.left = DataFrame( + {"key": Series(key, dtype=dtype), "value": np.random.randn(80000)} + ) + self.right = DataFrame( + { + "key": Series(indices[2000:], dtype=dtype), + "value2": np.random.randn(7999), + } + ) + if monotonic: + self.left = self.left.sort_values("key") + self.right = self.right.sort_values("key") + + def time_merge(self, dtype, monotonic): + merge(self.left, self.right) + + +class I8Merge: + params = ["inner", "outer", "left", "right"] + param_names = ["how"] + + def setup(self, how): + low, high, n = -1000, 1000, 10**6 + self.left = DataFrame( + np.random.randint(low, high, (n, 7)), columns=list("ABCDEFG") + ) + self.left["left"] = self.left.sum(axis=1) + self.right = self.left.sample(frac=1).rename({"left": "right"}, axis=1) + self.right = self.right.reset_index(drop=True) + self.right["right"] *= -1 + + def time_i8merge(self, how): + merge(self.left, self.right, how=how) + + +class UniqueMerge: + params = [4_000_000, 1_000_000] + param_names = ["unique_elements"] + + def setup(self, unique_elements): + N = 1_000_000 + self.left = DataFrame({"a": np.random.randint(1, unique_elements, (N,))}) + self.right = DataFrame({"a": np.random.randint(1, unique_elements, (N,))}) + uniques = self.right.a.drop_duplicates() + self.right["a"] = concat( + [uniques, Series(np.arange(0, -(N - len(uniques)), -1))], ignore_index=True + ) + + def time_unique_merge(self, unique_elements): + merge(self.left, self.right, how="inner") + + +class MergeDatetime: + params = [ + [ + ("ns", "ns"), + ("ms", "ms"), + ("ns", "ms"), + ], + [None, "Europe/Brussels"], + [True, False], + ] + param_names = ["units", "tz", "monotonic"] + + def setup(self, units, tz, monotonic): + unit_left, unit_right = units + N = 10_000 + keys = Series(date_range("2012-01-01", freq="min", periods=N, tz=tz)) + self.left = DataFrame( + { + "key": keys.sample(N * 10, replace=True).dt.as_unit(unit_left), + "value1": np.random.randn(N * 10), + } + ) + self.right = DataFrame( + { + "key": keys[:8000].dt.as_unit(unit_right), + "value2": np.random.randn(8000), + } + ) + if monotonic: + self.left = self.left.sort_values("key") + self.right = self.right.sort_values("key") + + def time_merge(self, units, tz, monotonic): + merge(self.left, self.right) + + +class MergeCategoricals: + def setup(self): + self.left_object = DataFrame( + { + "X": np.random.choice(range(10), size=(10000,)), + "Y": np.random.choice(["one", "two", "three"], size=(10000,)), + } + ) + + self.right_object = DataFrame( + { + "X": np.random.choice(range(10), size=(10000,)), + "Z": np.random.choice(["jjj", "kkk", "sss"], size=(10000,)), + } + ) + + self.left_cat = self.left_object.assign( + Y=self.left_object["Y"].astype("category") + ) + self.right_cat = self.right_object.assign( + Z=self.right_object["Z"].astype("category") + ) + + self.left_cat_col = self.left_object.astype({"X": "category"}) + self.right_cat_col = self.right_object.astype({"X": "category"}) + + self.left_cat_idx = self.left_cat_col.set_index("X") + self.right_cat_idx = self.right_cat_col.set_index("X") + + def time_merge_object(self): + merge(self.left_object, self.right_object, on="X") + + def time_merge_cat(self): + merge(self.left_cat, self.right_cat, on="X") + + def time_merge_on_cat_col(self): + merge(self.left_cat_col, self.right_cat_col, on="X") + + def time_merge_on_cat_idx(self): + merge(self.left_cat_idx, self.right_cat_idx, on="X") + + +class MergeOrdered: + def setup(self): + groups = Index([f"i-{i}" for i in range(10)], dtype=object).values + self.left = DataFrame( + { + "group": groups.repeat(5000), + "key": np.tile(np.arange(0, 10000, 2), 10), + "lvalue": np.random.randn(50000), + } + ) + self.right = DataFrame( + {"key": np.arange(10000), "rvalue": np.random.randn(10000)} + ) + + def time_merge_ordered(self): + merge_ordered(self.left, self.right, on="key", left_by="group") + + +class MergeAsof: + params = [["backward", "forward", "nearest"], [None, 5]] + param_names = ["direction", "tolerance"] + + def setup(self, direction, tolerance): + one_count = 200000 + two_count = 1000000 + + df1 = DataFrame( + { + "time": np.random.randint(0, one_count / 20, one_count), + "key": np.random.choice(list(string.ascii_uppercase), one_count), + "key2": np.random.randint(0, 25, one_count), + "value1": np.random.randn(one_count), + } + ) + df2 = DataFrame( + { + "time": np.random.randint(0, two_count / 20, two_count), + "key": np.random.choice(list(string.ascii_uppercase), two_count), + "key2": np.random.randint(0, 25, two_count), + "value2": np.random.randn(two_count), + } + ) + + df1 = df1.sort_values("time") + df2 = df2.sort_values("time") + + df1["time32"] = np.int32(df1.time) + df2["time32"] = np.int32(df2.time) + + df1["timeu64"] = np.uint64(df1.time) + df2["timeu64"] = np.uint64(df2.time) + + self.df1a = df1[["time", "value1"]] + self.df2a = df2[["time", "value2"]] + self.df1b = df1[["time", "key", "value1"]] + self.df2b = df2[["time", "key", "value2"]] + self.df1c = df1[["time", "key2", "value1"]] + self.df2c = df2[["time", "key2", "value2"]] + self.df1d = df1[["time32", "value1"]] + self.df2d = df2[["time32", "value2"]] + self.df1e = df1[["time", "key", "key2", "value1"]] + self.df2e = df2[["time", "key", "key2", "value2"]] + self.df1f = df1[["timeu64", "value1"]] + self.df2f = df2[["timeu64", "value2"]] + + def time_on_int(self, direction, tolerance): + merge_asof( + self.df1a, self.df2a, on="time", direction=direction, tolerance=tolerance + ) + + def time_on_int32(self, direction, tolerance): + merge_asof( + self.df1d, self.df2d, on="time32", direction=direction, tolerance=tolerance + ) + + def time_on_uint64(self, direction, tolerance): + merge_asof( + self.df1f, self.df2f, on="timeu64", direction=direction, tolerance=tolerance + ) + + def time_by_object(self, direction, tolerance): + merge_asof( + self.df1b, + self.df2b, + on="time", + by="key", + direction=direction, + tolerance=tolerance, + ) + + def time_by_int(self, direction, tolerance): + merge_asof( + self.df1c, + self.df2c, + on="time", + by="key2", + direction=direction, + tolerance=tolerance, + ) + + def time_multiby(self, direction, tolerance): + merge_asof( + self.df1e, + self.df2e, + on="time", + by=["key", "key2"], + direction=direction, + tolerance=tolerance, + ) + + +class MergeMultiIndex: + params = [ + [ + ("int64", "int64"), + ("datetime64[ns]", "int64"), + ("Int64", "Int64"), + ], + ["left", "right", "inner", "outer"], + ] + param_names = ["dtypes", "how"] + + def setup(self, dtypes, how): + n = 100_000 + offset = 50_000 + mi1 = MultiIndex.from_arrays( + [ + array(np.arange(n), dtype=dtypes[0]), + array(np.arange(n), dtype=dtypes[1]), + ] + ) + mi2 = MultiIndex.from_arrays( + [ + array(np.arange(offset, n + offset), dtype=dtypes[0]), + array(np.arange(offset, n + offset), dtype=dtypes[1]), + ] + ) + self.df1 = DataFrame({"col1": 1}, index=mi1) + self.df2 = DataFrame({"col2": 2}, index=mi2) + + def time_merge_sorted_multiindex(self, dtypes, how): + # copy to avoid MultiIndex._values caching + df1 = self.df1.copy() + df2 = self.df2.copy() + merge(df1, df2, how=how, left_index=True, right_index=True) + + +class Align: + def setup(self): + size = 5 * 10**5 + rng = np.arange(0, 10**13, 10**7) + stamps = np.datetime64("now").view("i8") + rng + idx1 = np.sort(np.random.choice(stamps, size, replace=False)) + idx2 = np.sort(np.random.choice(stamps, size, replace=False)) + self.ts1 = Series(np.random.randn(size), idx1) + self.ts2 = Series(np.random.randn(size), idx2) + + def time_series_align_int64_index(self): + self.ts1 + self.ts2 + + def time_series_align_left_monotonic(self): + self.ts1.align(self.ts2, join="left") + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/libs.py b/asv_bench/benchmarks/libs.py new file mode 100644 index 0000000000000..7da2d27d98dbb --- /dev/null +++ b/asv_bench/benchmarks/libs.py @@ -0,0 +1,105 @@ +""" +Benchmarks for code in pandas/_libs, excluding pandas/_libs/tslibs, +which has its own directory. + +If a PR does not edit anything in _libs/, then it is unlikely that the +benchmarks will be affected. +""" + +import numpy as np + +from pandas._libs.lib import ( + infer_dtype, + is_list_like, + is_scalar, +) + +from pandas import ( + NA, + Index, + NaT, +) + +from .pandas_vb_common import lib + +try: + from pandas.util import cache_readonly +except ImportError: + from pandas.util.decorators import cache_readonly + + +# TODO: share with something in pd._testing? +scalars = [ + 0, + 1.0, + 1 + 2j, + True, + "foo", + b"bar", + None, + np.datetime64(123, "ns"), + np.timedelta64(123, "ns"), + NaT, + NA, +] +zero_dims = [np.array("123")] +listlikes = [np.array([1, 2, 3]), {0: 1}, {1, 2, 3}, [1, 2, 3], (1, 2, 3)] + + +class ScalarListLike: + params = scalars + zero_dims + listlikes + + def time_is_list_like(self, param): + is_list_like(param) + + def time_is_scalar(self, param): + is_scalar(param) + + +class FastZip: + def setup(self): + N = 10000 + K = 10 + key1 = Index([f"i-{i}" for i in range(N)], dtype=object).values.repeat(K) + key2 = Index([f"i-{i}" for i in range(N)], dtype=object).values.repeat(K) + col_array = np.vstack([key1, key2, np.random.randn(N * K)]) + col_array2 = col_array.copy() + col_array2[:, :10000] = np.nan + self.col_array_list = list(col_array) + + def time_lib_fast_zip(self): + lib.fast_zip(self.col_array_list) + + +class InferDtype: + param_names = ["dtype"] + data_dict = { + "np-object": np.array([1] * 100000, dtype="O"), + "py-object": [1] * 100000, + "np-null": np.array([1] * 50000 + [np.nan] * 50000), + "py-null": [1] * 50000 + [None] * 50000, + "np-int": np.array([1] * 100000, dtype=int), + "np-floating": np.array([1.0] * 100000, dtype=float), + "empty": [], + "bytes": [b"a"] * 100000, + } + params = list(data_dict.keys()) + + def time_infer_dtype_skipna(self, dtype): + infer_dtype(self.data_dict[dtype], skipna=True) + + def time_infer_dtype(self, dtype): + infer_dtype(self.data_dict[dtype], skipna=False) + + +class CacheReadonly: + def setup(self): + class Foo: + @cache_readonly + def prop(self): + return 5 + + self.obj = Foo() + + def time_cache_readonly(self): + self.obj.prop diff --git a/asv_bench/benchmarks/multiindex_object.py b/asv_bench/benchmarks/multiindex_object.py new file mode 100644 index 0000000000000..0a588c9a2e22e --- /dev/null +++ b/asv_bench/benchmarks/multiindex_object.py @@ -0,0 +1,434 @@ +import string + +import numpy as np + +from pandas import ( + NA, + DataFrame, + Index, + MultiIndex, + RangeIndex, + Series, + array, + date_range, +) + + +class GetLoc: + def setup(self): + self.mi_large = MultiIndex.from_product( + [np.arange(1000), np.arange(20), list(string.ascii_letters)], + names=["one", "two", "three"], + ) + self.mi_med = MultiIndex.from_product( + [np.arange(1000), np.arange(10), list("A")], names=["one", "two", "three"] + ) + self.mi_small = MultiIndex.from_product( + [np.arange(100), list("A"), list("A")], names=["one", "two", "three"] + ) + + def time_large_get_loc(self): + self.mi_large.get_loc((999, 19, "Z")) + + def time_large_get_loc_warm(self): + for _ in range(1000): + self.mi_large.get_loc((999, 19, "Z")) + + def time_med_get_loc(self): + self.mi_med.get_loc((999, 9, "A")) + + def time_med_get_loc_warm(self): + for _ in range(1000): + self.mi_med.get_loc((999, 9, "A")) + + def time_string_get_loc(self): + self.mi_small.get_loc((99, "A", "A")) + + def time_small_get_loc_warm(self): + for _ in range(1000): + self.mi_small.get_loc((99, "A", "A")) + + +class GetLocs: + def setup(self): + self.mi_large = MultiIndex.from_product( + [np.arange(1000), np.arange(20), list(string.ascii_letters)], + names=["one", "two", "three"], + ) + self.mi_med = MultiIndex.from_product( + [np.arange(1000), np.arange(10), list("A")], names=["one", "two", "three"] + ) + self.mi_small = MultiIndex.from_product( + [np.arange(100), list("A"), list("A")], names=["one", "two", "three"] + ) + + def time_large_get_locs(self): + self.mi_large.get_locs([999, 19, "Z"]) + + def time_med_get_locs(self): + self.mi_med.get_locs([999, 9, "A"]) + + def time_small_get_locs(self): + self.mi_small.get_locs([99, "A", "A"]) + + +class Duplicates: + def setup(self): + size = 65536 + arrays = [np.random.randint(0, 8192, size), np.random.randint(0, 1024, size)] + mask = np.random.rand(size) < 0.1 + self.mi_unused_levels = MultiIndex.from_arrays(arrays) + self.mi_unused_levels = self.mi_unused_levels[mask] + + def time_remove_unused_levels(self): + self.mi_unused_levels.remove_unused_levels() + + +class Integer: + def setup(self): + self.mi_int = MultiIndex.from_product( + [np.arange(1000), np.arange(1000)], names=["one", "two"] + ) + self.obj_index = np.array( + [ + (0, 10), + (0, 11), + (0, 12), + (0, 13), + (0, 14), + (0, 15), + (0, 16), + (0, 17), + (0, 18), + (0, 19), + ], + dtype=object, + ) + self.other_mi_many_mismatches = MultiIndex.from_tuples( + [ + (-7, 41), + (-2, 3), + (-0.7, 5), + (0, 0), + (0, 1.5), + (0, 340), + (0, 1001), + (1, -4), + (1, 20), + (1, 1040), + (432, -5), + (432, 17), + (439, 165.5), + (998, -4), + (998, 24065), + (999, 865.2), + (999, 1000), + (1045, -843), + ] + ) + + def time_get_indexer(self): + self.mi_int.get_indexer(self.obj_index) + + def time_get_indexer_and_backfill(self): + self.mi_int.get_indexer(self.other_mi_many_mismatches, method="backfill") + + def time_get_indexer_and_pad(self): + self.mi_int.get_indexer(self.other_mi_many_mismatches, method="pad") + + def time_is_monotonic(self): + self.mi_int.is_monotonic_increasing + + +class Duplicated: + def setup(self): + n, k = 200, 5000 + levels = [ + np.arange(n), + Index([f"i-{i}" for i in range(n)], dtype=object).values, + 1000 + np.arange(n), + ] + codes = [np.random.choice(n, (k * n)) for lev in levels] + self.mi = MultiIndex(levels=levels, codes=codes) + + def time_duplicated(self): + self.mi.duplicated() + + +class Sortlevel: + def setup(self): + n = 1182720 + low, high = -4096, 4096 + arrs = [ + np.repeat(np.random.randint(low, high, (n // k)), k) + for k in [11, 7, 5, 3, 1] + ] + self.mi_int = MultiIndex.from_arrays(arrs)[np.random.permutation(n)] + + a = np.repeat(np.arange(100), 1000) + b = np.tile(np.arange(1000), 100) + self.mi = MultiIndex.from_arrays([a, b]) + self.mi = self.mi.take(np.random.permutation(np.arange(100000))) + + def time_sortlevel_int64(self): + self.mi_int.sortlevel() + + def time_sortlevel_zero(self): + self.mi.sortlevel(0) + + def time_sortlevel_one(self): + self.mi.sortlevel(1) + + +class SortValues: + params = ["int64", "Int64"] + param_names = ["dtype"] + + def setup(self, dtype): + a = array(np.tile(np.arange(100), 1000), dtype=dtype) + b = array(np.tile(np.arange(1000), 100), dtype=dtype) + self.mi = MultiIndex.from_arrays([a, b]) + + def time_sort_values(self, dtype): + self.mi.sort_values() + + +class Values: + def setup_cache(self): + level1 = range(1000) + level2 = date_range(start="1/1/2012", periods=100) + mi = MultiIndex.from_product([level1, level2]) + return mi + + def time_datetime_level_values_copy(self, mi): + mi.copy().values + + def time_datetime_level_values_sliced(self, mi): + mi[:10].values + + +class CategoricalLevel: + def setup(self): + self.df = DataFrame( + { + "a": np.arange(1_000_000, dtype=np.int32), + "b": np.arange(1_000_000, dtype=np.int64), + "c": np.arange(1_000_000, dtype=float), + } + ).astype({"a": "category", "b": "category"}) + + def time_categorical_level(self): + self.df.set_index(["a", "b"]) + + +class Equals: + def setup(self): + self.mi = MultiIndex.from_product( + [ + date_range("2000-01-01", periods=1000), + RangeIndex(1000), + ] + ) + self.mi_deepcopy = self.mi.copy(deep=True) + self.idx_non_object = RangeIndex(1) + + def time_equals_deepcopy(self): + self.mi.equals(self.mi_deepcopy) + + def time_equals_non_object_index(self): + self.mi.equals(self.idx_non_object) + + +class SetOperations: + params = [ + ("monotonic", "non_monotonic"), + ("datetime", "int", "string", "ea_int"), + ("intersection", "union", "symmetric_difference"), + (False, None), + ] + param_names = ["index_structure", "dtype", "method", "sort"] + + def setup(self, index_structure, dtype, method, sort): + N = 10**5 + level1 = range(1000) + + level2 = date_range(start="1/1/2000", periods=N // 1000) + dates_left = MultiIndex.from_product([level1, level2]) + + level2 = range(N // 1000) + int_left = MultiIndex.from_product([level1, level2]) + + level2 = Index([f"i-{i}" for i in range(N // 1000)], dtype=object).values + str_left = MultiIndex.from_product([level1, level2]) + + level2 = range(N // 1000) + ea_int_left = MultiIndex.from_product([level1, Series(level2, dtype="Int64")]) + + data = { + "datetime": dates_left, + "int": int_left, + "string": str_left, + "ea_int": ea_int_left, + } + + if index_structure == "non_monotonic": + data = {k: mi[::-1] for k, mi in data.items()} + + data = {k: {"left": mi, "right": mi[:-1]} for k, mi in data.items()} + self.left = data[dtype]["left"] + self.right = data[dtype]["right"] + + def time_operation(self, index_structure, dtype, method, sort): + getattr(self.left, method)(self.right, sort=sort) + + +class Difference: + params = [ + ("datetime", "int", "string", "ea_int"), + ] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10**4 * 2 + level1 = range(1000) + + level2 = date_range(start="1/1/2000", periods=N // 1000) + dates_left = MultiIndex.from_product([level1, level2]) + + level2 = range(N // 1000) + int_left = MultiIndex.from_product([level1, level2]) + + level2 = Series(range(N // 1000), dtype="Int64") + level2[0] = NA + ea_int_left = MultiIndex.from_product([level1, level2]) + + level2 = Index([f"i-{i}" for i in range(N // 1000)], dtype=object).values + str_left = MultiIndex.from_product([level1, level2]) + + data = { + "datetime": dates_left, + "int": int_left, + "ea_int": ea_int_left, + "string": str_left, + } + + data = {k: {"left": mi, "right": mi[:5]} for k, mi in data.items()} + self.left = data[dtype]["left"] + self.right = data[dtype]["right"] + + def time_difference(self, dtype): + self.left.difference(self.right) + + +class Unique: + params = [ + (("Int64", NA), ("int64", 0)), + ] + param_names = ["dtype_val"] + + def setup(self, dtype_val): + level = Series( + [1, 2, dtype_val[1], dtype_val[1]] + list(range(1_000_000)), + dtype=dtype_val[0], + ) + self.midx = MultiIndex.from_arrays([level, level]) + + level_dups = Series( + [1, 2, dtype_val[1], dtype_val[1]] + list(range(500_000)) * 2, + dtype=dtype_val[0], + ) + + self.midx_dups = MultiIndex.from_arrays([level_dups, level_dups]) + + def time_unique(self, dtype_val): + self.midx.unique() + + def time_unique_dups(self, dtype_val): + self.midx_dups.unique() + + +class Isin: + params = [ + ("string", "int", "datetime"), + ] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10**5 + level1 = range(1000) + + level2 = date_range(start="1/1/2000", periods=N // 1000) + dates_midx = MultiIndex.from_product([level1, level2]) + + level2 = range(N // 1000) + int_midx = MultiIndex.from_product([level1, level2]) + + level2 = Index([f"i-{i}" for i in range(N // 1000)], dtype=object).values + str_midx = MultiIndex.from_product([level1, level2]) + + data = { + "datetime": dates_midx, + "int": int_midx, + "string": str_midx, + } + + self.midx = data[dtype] + self.values_small = self.midx[:100] + self.values_large = self.midx[100:] + + def time_isin_small(self, dtype): + self.midx.isin(self.values_small) + + def time_isin_large(self, dtype): + self.midx.isin(self.values_large) + + +class Putmask: + def setup(self): + N = 10**5 + level1 = range(1_000) + + level2 = date_range(start="1/1/2000", periods=N // 1000) + self.midx = MultiIndex.from_product([level1, level2]) + + level1 = range(1_000, 2_000) + self.midx_values = MultiIndex.from_product([level1, level2]) + + level2 = date_range(start="1/1/2010", periods=N // 1000) + self.midx_values_different = MultiIndex.from_product([level1, level2]) + self.mask = np.array([True, False] * (N // 2)) + + def time_putmask(self): + self.midx.putmask(self.mask, self.midx_values) + + def time_putmask_all_different(self): + self.midx.putmask(self.mask, self.midx_values_different) + + +class Append: + params = ["datetime64[ns]", "int64", "string"] + param_names = ["dtype"] + + def setup(self, dtype): + N1 = 1000 + N2 = 500 + left_level1 = range(N1) + right_level1 = range(N1, N1 + N1) + + if dtype == "datetime64[ns]": + level2 = date_range(start="2000-01-01", periods=N2) + elif dtype == "int64": + level2 = range(N2) + elif dtype == "string": + level2 = Index([f"i-{i}" for i in range(N2)], dtype=object) + else: + raise NotImplementedError + + self.left = MultiIndex.from_product([left_level1, level2]) + self.right = MultiIndex.from_product([right_level1, level2]) + + def time_append(self, dtype): + self.left.append(self.right) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/package.py b/asv_bench/benchmarks/package.py new file mode 100644 index 0000000000000..f8b51a523dab8 --- /dev/null +++ b/asv_bench/benchmarks/package.py @@ -0,0 +1,20 @@ +""" +Benchmarks for pandas at the package-level. +""" + +import subprocess +import sys + + +class TimeImport: + def time_import(self): + # on py37+ we the "-X importtime" usage gives us a more precise + # measurement of the import time we actually care about, + # without the subprocess or interpreter overhead + cmd = [sys.executable, "-X", "importtime", "-c", "import pandas as pd"] + p = subprocess.run(cmd, stderr=subprocess.PIPE, check=True) + + line = p.stderr.splitlines()[-1] + field = line.split(b"|")[-2].strip() + total = int(field) # microseconds + return total diff --git a/asv_bench/benchmarks/pandas_vb_common.py b/asv_bench/benchmarks/pandas_vb_common.py new file mode 100644 index 0000000000000..4bd56ccb1b5ce --- /dev/null +++ b/asv_bench/benchmarks/pandas_vb_common.py @@ -0,0 +1,80 @@ +from importlib import import_module +import os + +import numpy as np + +import pandas as pd + +# Compatibility import for lib +for imp in ["pandas._libs.lib", "pandas.lib"]: + try: + lib = import_module(imp) + break + except (ImportError, TypeError, ValueError): + pass + +# Compatibility import for the testing module +try: + import pandas._testing as tm +except ImportError: + import pandas.util.testing as tm # noqa: F401 + + +numeric_dtypes = [ + np.int64, + np.int32, + np.uint32, + np.uint64, + np.float32, + np.float64, + np.int16, + np.int8, + np.uint16, + np.uint8, +] +datetime_dtypes = [np.datetime64, np.timedelta64] +string_dtypes = [object] +try: + extension_dtypes = [ + pd.Int8Dtype, + pd.Int16Dtype, + pd.Int32Dtype, + pd.Int64Dtype, + pd.UInt8Dtype, + pd.UInt16Dtype, + pd.UInt32Dtype, + pd.UInt64Dtype, + pd.CategoricalDtype, + pd.IntervalDtype, + pd.DatetimeTZDtype("ns", "UTC"), + pd.PeriodDtype("D"), + ] +except AttributeError: + extension_dtypes = [] + + +def setup(*args, **kwargs): + # This function just needs to be imported into each benchmark file to + # set up the random seed before each function. + # https://asv.readthedocs.io/en/latest/writing_benchmarks.html + np.random.seed(1234) + + +class BaseIO: + """ + Base class for IO benchmarks + """ + + fname = None + + def remove(self, f): + """Remove created files""" + try: + os.remove(f) + except OSError: + # On Windows, attempting to remove a file that is in use + # causes an exception to be raised + pass + + def teardown(self, *args, **kwargs): + self.remove(self.fname) diff --git a/asv_bench/benchmarks/period.py b/asv_bench/benchmarks/period.py new file mode 100644 index 0000000000000..3b8b60e790380 --- /dev/null +++ b/asv_bench/benchmarks/period.py @@ -0,0 +1,107 @@ +""" +Period benchmarks with non-tslibs dependencies. See +benchmarks.tslibs.period for benchmarks that rely only on tslibs. +""" + +from pandas import ( + DataFrame, + Period, + PeriodIndex, + Series, + date_range, + period_range, +) + +from pandas.tseries.frequencies import to_offset + + +class PeriodIndexConstructor: + params = [["D"], [True, False]] + param_names = ["freq", "is_offset"] + + def setup(self, freq, is_offset): + self.rng = date_range("1985", periods=1000) + self.rng2 = date_range("1985", periods=1000).to_pydatetime() + self.ints = list(range(2000, 3000)) + self.daily_ints = ( + date_range("1/1/2000", periods=1000, freq=freq).strftime("%Y%m%d").map(int) + ) + if is_offset: + self.freq = to_offset(freq) + else: + self.freq = freq + + def time_from_date_range(self, freq, is_offset): + PeriodIndex(self.rng, freq=freq) + + def time_from_pydatetime(self, freq, is_offset): + PeriodIndex(self.rng2, freq=freq) + + def time_from_ints(self, freq, is_offset): + PeriodIndex(self.ints, freq=freq) + + def time_from_ints_daily(self, freq, is_offset): + PeriodIndex(self.daily_ints, freq=freq) + + +class DataFramePeriodColumn: + def setup(self): + self.rng = period_range(start="1/1/1990", freq="s", periods=20000) + self.df = DataFrame(index=range(len(self.rng))) + + def time_setitem_period_column(self): + self.df["col"] = self.rng + + def time_set_index(self): + # GH#21582 limited by comparisons of Period objects + self.df["col2"] = self.rng + self.df.set_index("col2", append=True) + + +class Algorithms: + params = ["index", "series"] + param_names = ["typ"] + + def setup(self, typ): + data = [ + Period("2011-01", freq="M"), + Period("2011-02", freq="M"), + Period("2011-03", freq="M"), + Period("2011-04", freq="M"), + ] + + if typ == "index": + self.vector = PeriodIndex(data * 1000, freq="M") + elif typ == "series": + self.vector = Series(data * 1000) + + def time_drop_duplicates(self, typ): + self.vector.drop_duplicates() + + def time_value_counts(self, typ): + self.vector.value_counts() + + +class Indexing: + def setup(self): + self.index = period_range(start="1985", periods=1000, freq="D") + self.series = Series(range(1000), index=self.index) + self.period = self.index[500] + + def time_get_loc(self): + self.index.get_loc(self.period) + + def time_shallow_copy(self): + self.index._view() + + def time_series_loc(self): + self.series.loc[self.period] + + def time_align(self): + DataFrame({"a": self.series, "b": self.series[:500]}) + + def time_intersection(self): + self.index[:750].intersection(self.index[250:]) + + def time_unique(self): + self.index.unique() diff --git a/asv_bench/benchmarks/plotting.py b/asv_bench/benchmarks/plotting.py new file mode 100644 index 0000000000000..789bb8d8533b1 --- /dev/null +++ b/asv_bench/benchmarks/plotting.py @@ -0,0 +1,164 @@ +import contextlib +import importlib.machinery +import importlib.util +import os +import pathlib +import sys +import tempfile +from unittest import mock + +import matplotlib +import numpy as np + +from pandas import ( + DataFrame, + DatetimeIndex, + Series, + date_range, +) + +try: + from pandas.plotting import andrews_curves +except ImportError: + from pandas.tools.plotting import andrews_curves + +from pandas.plotting._core import _get_plot_backend + +matplotlib.use("Agg") + + +class SeriesPlotting: + params = [["line", "bar", "area", "barh", "hist", "kde", "pie"]] + param_names = ["kind"] + + def setup(self, kind): + if kind in ["bar", "barh", "pie"]: + n = 100 + elif kind in ["kde"]: + n = 10000 + else: + n = 1000000 + + self.s = Series(np.random.randn(n)) + if kind in ["area", "pie"]: + self.s = self.s.abs() + + def time_series_plot(self, kind): + self.s.plot(kind=kind) + + +class FramePlotting: + params = [ + ["line", "bar", "area", "barh", "hist", "kde", "pie", "scatter", "hexbin"] + ] + param_names = ["kind"] + + def setup(self, kind): + if kind in ["bar", "barh", "pie"]: + n = 100 + elif kind in ["kde", "scatter", "hexbin"]: + n = 10000 + else: + n = 1000000 + + self.x = Series(np.random.randn(n)) + self.y = Series(np.random.randn(n)) + if kind in ["area", "pie"]: + self.x = self.x.abs() + self.y = self.y.abs() + self.df = DataFrame({"x": self.x, "y": self.y}) + + def time_frame_plot(self, kind): + self.df.plot(x="x", y="y", kind=kind) + + +class TimeseriesPlotting: + def setup(self): + N = 2000 + M = 5 + idx = date_range("1/1/1975", periods=N) + self.df = DataFrame(np.random.randn(N, M), index=idx) + + idx_irregular = DatetimeIndex( + np.concatenate((idx.values[0:10], idx.values[12:])) + ) + self.df2 = DataFrame( + np.random.randn(len(idx_irregular), M), index=idx_irregular + ) + + def time_plot_regular(self): + self.df.plot() + + def time_plot_regular_compat(self): + self.df.plot(x_compat=True) + + def time_plot_irregular(self): + self.df2.plot() + + def time_plot_table(self): + self.df.plot(table=True) + + +class Misc: + def setup(self): + N = 500 + M = 10 + self.df = DataFrame(np.random.randn(N, M)) + self.df["Name"] = ["A"] * N + + def time_plot_andrews_curves(self): + andrews_curves(self.df, "Name") + + +class BackendLoading: + repeat = 1 + number = 1 + warmup_time = 0 + + def setup(self): + mod = importlib.util.module_from_spec( + importlib.machinery.ModuleSpec("pandas_dummy_backend", None) + ) + mod.plot = lambda *args, **kwargs: 1 + + with contextlib.ExitStack() as stack: + stack.enter_context( + mock.patch.dict(sys.modules, {"pandas_dummy_backend": mod}) + ) + tmp_path = pathlib.Path(stack.enter_context(tempfile.TemporaryDirectory())) + + sys.path.insert(0, os.fsdecode(tmp_path)) + stack.callback(sys.path.remove, os.fsdecode(tmp_path)) + + dist_info = tmp_path / "my_backend-0.0.0.dist-info" + dist_info.mkdir() + (dist_info / "entry_points.txt").write_bytes( + b"[pandas_plotting_backends]\n" + b"my_ep_backend = pandas_dummy_backend\n" + b"my_ep_backend0 = pandas_dummy_backend\n" + b"my_ep_backend1 = pandas_dummy_backend\n" + b"my_ep_backend2 = pandas_dummy_backend\n" + b"my_ep_backend3 = pandas_dummy_backend\n" + b"my_ep_backend4 = pandas_dummy_backend\n" + b"my_ep_backend5 = pandas_dummy_backend\n" + b"my_ep_backend6 = pandas_dummy_backend\n" + b"my_ep_backend7 = pandas_dummy_backend\n" + b"my_ep_backend8 = pandas_dummy_backend\n" + b"my_ep_backend9 = pandas_dummy_backend\n" + ) + self.stack = stack.pop_all() + + def teardown(self): + self.stack.close() + + def time_get_plot_backend(self): + # finds the first my_ep_backend + _get_plot_backend("my_ep_backend") + + def time_get_plot_backend_fallback(self): + # iterates through all the my_ep_backend[0-9] before falling back + # to importlib.import_module + _get_plot_backend("pandas_dummy_backend") + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/reindex.py b/asv_bench/benchmarks/reindex.py new file mode 100644 index 0000000000000..3d22bfce7e2b2 --- /dev/null +++ b/asv_bench/benchmarks/reindex.py @@ -0,0 +1,148 @@ +import numpy as np + +from pandas import ( + DataFrame, + Index, + MultiIndex, + Series, + date_range, + period_range, +) + + +class Reindex: + def setup(self): + rng = date_range(start="1/1/1970", periods=10000, freq="1min") + self.df = DataFrame(np.random.rand(10000, 10), index=rng, columns=range(10)) + self.df["foo"] = "bar" + self.rng_subset = Index(rng[::2]) + self.df2 = DataFrame( + index=range(10000), data=np.random.rand(10000, 30), columns=range(30) + ) + N = 5000 + K = 200 + level1 = Index([f"i-{i}" for i in range(N)], dtype=object).values.repeat(K) + level2 = np.tile(Index([f"i-{i}" for i in range(K)], dtype=object).values, N) + index = MultiIndex.from_arrays([level1, level2]) + self.s = Series(np.random.randn(N * K), index=index) + self.s_subset = self.s[::2] + self.s_subset_no_cache = self.s[::2].copy() + + mi = MultiIndex.from_product([rng, range(100)]) + self.s2 = Series(np.random.randn(len(mi)), index=mi) + self.s2_subset = self.s2[::2].copy() + + def time_reindex_dates(self): + self.df.reindex(self.rng_subset) + + def time_reindex_columns(self): + self.df2.reindex(columns=self.df.columns[1:5]) + + def time_reindex_multiindex_with_cache(self): + # MultiIndex._values gets cached + self.s.reindex(self.s_subset.index) + + def time_reindex_multiindex_no_cache(self): + # Copy to avoid MultiIndex._values getting cached + self.s.reindex(self.s_subset_no_cache.index.copy()) + + def time_reindex_multiindex_no_cache_dates(self): + # Copy to avoid MultiIndex._values getting cached + self.s2_subset.reindex(self.s2.index.copy()) + + +class ReindexMethod: + params = [["pad", "backfill"], [date_range, period_range]] + param_names = ["method", "constructor"] + + def setup(self, method, constructor): + N = 100000 + self.idx = constructor("1/1/2000", periods=N, freq="1min") + self.ts = Series(np.random.randn(N), index=self.idx)[::2] + + def time_reindex_method(self, method, constructor): + self.ts.reindex(self.idx, method=method) + + +class LevelAlign: + def setup(self): + self.index = MultiIndex( + levels=[np.arange(10), np.arange(100), np.arange(100)], + codes=[ + np.arange(10).repeat(10000), + np.tile(np.arange(100).repeat(100), 10), + np.tile(np.tile(np.arange(100), 100), 10), + ], + ) + self.df = DataFrame(np.random.randn(len(self.index), 4), index=self.index) + self.df_level = DataFrame(np.random.randn(100, 4), index=self.index.levels[1]) + + def time_align_level(self): + self.df.align(self.df_level, level=1, copy=False) + + def time_reindex_level(self): + self.df_level.reindex(self.index, level=1) + + +class DropDuplicates: + params = [True, False] + param_names = ["inplace"] + + def setup(self, inplace): + N = 10000 + K = 10 + key1 = Index([f"i-{i}" for i in range(N)], dtype=object).values.repeat(K) + key2 = Index([f"i-{i}" for i in range(N)], dtype=object).values.repeat(K) + self.df = DataFrame( + {"key1": key1, "key2": key2, "value": np.random.randn(N * K)} + ) + self.df_nan = self.df.copy() + self.df_nan.iloc[:10000, :] = np.nan + + self.s = Series(np.random.randint(0, 1000, size=10000)) + self.s_str = Series( + np.tile(Index([f"i-{i}" for i in range(1000)], dtype=object).values, 10) + ) + + N = 1000000 + K = 10000 + key1 = np.random.randint(0, K, size=N) + self.df_int = DataFrame({"key1": key1}) + self.df_bool = DataFrame(np.random.randint(0, 2, size=(K, 10), dtype=bool)) + + def time_frame_drop_dups(self, inplace): + self.df.drop_duplicates(["key1", "key2"], inplace=inplace) + + def time_frame_drop_dups_na(self, inplace): + self.df_nan.drop_duplicates(["key1", "key2"], inplace=inplace) + + def time_series_drop_dups_int(self, inplace): + self.s.drop_duplicates(inplace=inplace) + + def time_series_drop_dups_string(self, inplace): + self.s_str.drop_duplicates(inplace=inplace) + + def time_frame_drop_dups_int(self, inplace): + self.df_int.drop_duplicates(inplace=inplace) + + def time_frame_drop_dups_bool(self, inplace): + self.df_bool.drop_duplicates(inplace=inplace) + + +class Align: + # blog "pandas escaped the zoo" + def setup(self): + n = 50000 + indices = Index([f"i-{i}" for i in range(n)], dtype=object) + subsample_size = 40000 + self.x = Series(np.random.randn(n), indices) + self.y = Series( + np.random.randn(subsample_size), + index=np.random.choice(indices, subsample_size, replace=False), + ) + + def time_align_series_irregular_string(self): + self.x + self.y + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/replace.py b/asv_bench/benchmarks/replace.py new file mode 100644 index 0000000000000..a9276b7dc32ce --- /dev/null +++ b/asv_bench/benchmarks/replace.py @@ -0,0 +1,74 @@ +import numpy as np + +import pandas as pd + + +class FillNa: + params = [True, False] + param_names = ["inplace"] + + def setup(self, inplace): + N = 10**6 + rng = pd.date_range("1/1/2000", periods=N, freq="min") + data = np.random.randn(N) + data[::2] = np.nan + self.ts = pd.Series(data, index=rng) + + def time_fillna(self, inplace): + self.ts.fillna(0.0, inplace=inplace) + + def time_replace(self, inplace): + self.ts.replace(np.nan, 0.0, inplace=inplace) + + +class ReplaceDict: + params = [True, False] + param_names = ["inplace"] + + def setup(self, inplace): + N = 10**5 + start_value = 10**5 + self.to_rep = dict(enumerate(np.arange(N) + start_value)) + self.s = pd.Series(np.random.randint(N, size=10**3)) + + def time_replace_series(self, inplace): + self.s.replace(self.to_rep, inplace=inplace) + + +class ReplaceList: + # GH#28099 + + params = [(True, False)] + param_names = ["inplace"] + + def setup(self, inplace): + self.df = pd.DataFrame({"A": 0, "B": 0}, index=range(10**7)) + + def time_replace_list(self, inplace): + self.df.replace([np.inf, -np.inf], np.nan, inplace=inplace) + + def time_replace_list_one_match(self, inplace): + # the 1 can be held in self._df.blocks[0], while the inf and -inf can't + self.df.replace([np.inf, -np.inf, 1], np.nan, inplace=inplace) + + +class Convert: + params = (["DataFrame", "Series"], ["Timestamp", "Timedelta"]) + param_names = ["constructor", "replace_data"] + + def setup(self, constructor, replace_data): + N = 10**3 + data = { + "Series": pd.Series(np.random.randint(N, size=N)), + "DataFrame": pd.DataFrame( + {"A": np.random.randint(N, size=N), "B": np.random.randint(N, size=N)} + ), + } + self.to_replace = {i: getattr(pd, replace_data) for i in range(N)} + self.data = data[constructor] + + def time_replace(self, constructor, replace_data): + self.data.replace(self.to_replace) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/reshape.py b/asv_bench/benchmarks/reshape.py new file mode 100644 index 0000000000000..54326a4433756 --- /dev/null +++ b/asv_bench/benchmarks/reshape.py @@ -0,0 +1,338 @@ +from itertools import product +import string + +import numpy as np + +import pandas as pd +from pandas import ( + DataFrame, + MultiIndex, + date_range, + melt, + wide_to_long, +) +from pandas.api.types import CategoricalDtype + + +class Melt: + params = ["float64", "Float64"] + param_names = ["dtype"] + + def setup(self, dtype): + self.df = DataFrame( + np.random.randn(100_000, 3), columns=["A", "B", "C"], dtype=dtype + ) + self.df["id1"] = pd.Series(np.random.randint(0, 10, 10000)) + self.df["id2"] = pd.Series(np.random.randint(100, 1000, 10000)) + + def time_melt_dataframe(self, dtype): + melt(self.df, id_vars=["id1", "id2"]) + + +class Pivot: + def setup(self): + N = 10000 + index = date_range("1/1/2000", periods=N, freq="h") + data = { + "value": np.random.randn(N * 50), + "variable": np.arange(50).repeat(N), + "date": np.tile(index.values, 50), + } + self.df = DataFrame(data) + + def time_reshape_pivot_time_series(self): + self.df.pivot(index="date", columns="variable", values="value") + + +class SimpleReshape: + def setup(self): + arrays = [np.arange(100).repeat(100), np.roll(np.tile(np.arange(100), 100), 25)] + index = MultiIndex.from_arrays(arrays) + self.df = DataFrame(np.random.randn(10000, 4), index=index) + self.udf = self.df.unstack(1) + + def time_stack(self): + self.udf.stack() + + def time_unstack(self): + self.df.unstack(1) + + +class ReshapeExtensionDtype: + params = ["datetime64[ns, US/Pacific]", "Period[s]"] + param_names = ["dtype"] + + def setup(self, dtype): + lev = pd.Index(list("ABCDEFGHIJ")) + ri = pd.Index(range(1000)) + mi = MultiIndex.from_product([lev, ri], names=["foo", "bar"]) + + index = date_range("2016-01-01", periods=10000, freq="s", tz="US/Pacific") + if dtype == "Period[s]": + index = index.tz_localize(None).to_period("s") + + ser = pd.Series(index, index=mi) + df = ser.unstack("bar") + # roundtrips -> df.stack().equals(ser) + + self.ser = ser + self.df = df + + def time_stack(self, dtype): + self.df.stack() + + def time_unstack_fast(self, dtype): + # last level -> doesn't have to make copies + self.ser.unstack("bar") + + def time_unstack_slow(self, dtype): + # first level -> must make copies + self.ser.unstack("foo") + + def time_transpose(self, dtype): + self.df.T + + +class ReshapeMaskedArrayDtype(ReshapeExtensionDtype): + params = ["Int64", "Float64"] + param_names = ["dtype"] + + def setup(self, dtype): + lev = pd.Index(list("ABCDEFGHIJ")) + ri = pd.Index(range(1000)) + mi = MultiIndex.from_product([lev, ri], names=["foo", "bar"]) + + values = np.random.randn(10_000).astype(int) + + ser = pd.Series(values, dtype=dtype, index=mi) + df = ser.unstack("bar") + # roundtrips -> df.stack().equals(ser) + + self.ser = ser + self.df = df + + +class Unstack: + params = ["int", "category"] + + def setup(self, dtype): + m = 100 + n = 1000 + + levels = np.arange(m) + index = MultiIndex.from_product([levels] * 2) + columns = np.arange(n) + if dtype == "int": + values = np.arange(m * m * n).reshape(m * m, n) + self.df = DataFrame(values, index, columns) + else: + # the category branch is ~20x slower than int. So we + # cut down the size a bit. Now it's only ~3x slower. + n = 50 + columns = columns[:n] + indices = np.random.randint(0, 52, size=(m * m, n)) + values = np.take(list(string.ascii_letters), indices) + values = [pd.Categorical(v) for v in values.T] + + self.df = DataFrame(dict(enumerate(values)), index, columns) + + self.df2 = self.df.iloc[:-1] + + def time_full_product(self, dtype): + self.df.unstack() + + def time_without_last_row(self, dtype): + self.df2.unstack() + + +class SparseIndex: + def setup(self): + NUM_ROWS = 1000 + self.df = DataFrame( + { + "A": np.random.randint(50, size=NUM_ROWS), + "B": np.random.randint(50, size=NUM_ROWS), + "C": np.random.randint(-10, 10, size=NUM_ROWS), + "D": np.random.randint(-10, 10, size=NUM_ROWS), + "E": np.random.randint(10, size=NUM_ROWS), + "F": np.random.randn(NUM_ROWS), + } + ) + self.df = self.df.set_index(["A", "B", "C", "D", "E"]) + + def time_unstack(self): + self.df.unstack() + + +class WideToLong: + def setup(self): + nyrs = 20 + nidvars = 20 + N = 5000 + self.letters = list("ABCD") + yrvars = [ + letter + str(num) + for letter, num in product(self.letters, range(1, nyrs + 1)) + ] + columns = [str(i) for i in range(nidvars)] + yrvars + self.df = DataFrame(np.random.randn(N, nidvars + len(yrvars)), columns=columns) + self.df["id"] = self.df.index + + def time_wide_to_long_big(self): + wide_to_long(self.df, self.letters, i="id", j="year") + + +class PivotTable: + def setup(self): + N = 100000 + fac1 = np.array(["A", "B", "C"], dtype="O") + fac2 = np.array(["one", "two"], dtype="O") + ind1 = np.random.randint(0, 3, size=N) + ind2 = np.random.randint(0, 2, size=N) + self.df = DataFrame( + { + "key1": fac1.take(ind1), + "key2": fac2.take(ind2), + "key3": fac2.take(ind2), + "value1": np.random.randn(N), + "value2": np.random.randn(N), + "value3": np.random.randn(N), + } + ) + self.df2 = DataFrame( + {"col1": list("abcde"), "col2": list("fghij"), "col3": [1, 2, 3, 4, 5]} + ) + self.df2.col1 = self.df2.col1.astype("category") + self.df2.col2 = self.df2.col2.astype("category") + + def time_pivot_table(self): + self.df.pivot_table(index="key1", columns=["key2", "key3"]) + + def time_pivot_table_agg(self): + self.df.pivot_table( + index="key1", columns=["key2", "key3"], aggfunc=["sum", "mean"] + ) + + def time_pivot_table_margins(self): + self.df.pivot_table(index="key1", columns=["key2", "key3"], margins=True) + + def time_pivot_table_categorical(self): + self.df2.pivot_table( + index="col1", values="col3", columns="col2", aggfunc="sum", fill_value=0 + ) + + def time_pivot_table_categorical_observed(self): + self.df2.pivot_table( + index="col1", + values="col3", + columns="col2", + aggfunc="sum", + fill_value=0, + observed=True, + ) + + def time_pivot_table_margins_only_column(self): + self.df.pivot_table(columns=["key1", "key2", "key3"], margins=True) + + +class Crosstab: + def setup(self): + N = 100000 + fac1 = np.array(["A", "B", "C"], dtype="O") + fac2 = np.array(["one", "two"], dtype="O") + self.ind1 = np.random.randint(0, 3, size=N) + self.ind2 = np.random.randint(0, 2, size=N) + self.vec1 = fac1.take(self.ind1) + self.vec2 = fac2.take(self.ind2) + + def time_crosstab(self): + pd.crosstab(self.vec1, self.vec2) + + def time_crosstab_values(self): + pd.crosstab(self.vec1, self.vec2, values=self.ind1, aggfunc="sum") + + def time_crosstab_normalize(self): + pd.crosstab(self.vec1, self.vec2, normalize=True) + + def time_crosstab_normalize_margins(self): + pd.crosstab(self.vec1, self.vec2, normalize=True, margins=True) + + +class GetDummies: + def setup(self): + categories = list(string.ascii_letters[:12]) + s = pd.Series( + np.random.choice(categories, size=1000000), + dtype=CategoricalDtype(categories), + ) + self.s = s + + def time_get_dummies_1d(self): + pd.get_dummies(self.s, sparse=False) + + def time_get_dummies_1d_sparse(self): + pd.get_dummies(self.s, sparse=True) + + +class Cut: + params = [[4, 10, 1000]] + param_names = ["bins"] + + def setup(self, bins): + N = 10**5 + self.int_series = pd.Series(np.arange(N).repeat(5)) + self.float_series = pd.Series(np.random.randn(N).repeat(5)) + self.timedelta_series = pd.Series( + np.random.randint(N, size=N), dtype="timedelta64[ns]" + ) + self.datetime_series = pd.Series( + np.random.randint(N, size=N), dtype="datetime64[ns]" + ) + self.interval_bins = pd.IntervalIndex.from_breaks(np.linspace(0, N, bins)) + + def time_cut_int(self, bins): + pd.cut(self.int_series, bins) + + def time_cut_float(self, bins): + pd.cut(self.float_series, bins) + + def time_cut_timedelta(self, bins): + pd.cut(self.timedelta_series, bins) + + def time_cut_datetime(self, bins): + pd.cut(self.datetime_series, bins) + + def time_qcut_int(self, bins): + pd.qcut(self.int_series, bins) + + def time_qcut_float(self, bins): + pd.qcut(self.float_series, bins) + + def time_qcut_timedelta(self, bins): + pd.qcut(self.timedelta_series, bins) + + def time_qcut_datetime(self, bins): + pd.qcut(self.datetime_series, bins) + + def time_cut_interval(self, bins): + # GH 27668 + pd.cut(self.int_series, self.interval_bins) + + def peakmem_cut_interval(self, bins): + # GH 27668 + pd.cut(self.int_series, self.interval_bins) + + +class Explode: + param_names = ["n_rows", "max_list_length"] + params = [[100, 1000, 10000], [3, 5, 10]] + + def setup(self, n_rows, max_list_length): + data = [np.arange(np.random.randint(max_list_length)) for _ in range(n_rows)] + self.series = pd.Series(data) + + def time_explode(self, n_rows, max_list_length): + self.series.explode() + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/rolling.py b/asv_bench/benchmarks/rolling.py new file mode 100644 index 0000000000000..f9a5f38c2e349 --- /dev/null +++ b/asv_bench/benchmarks/rolling.py @@ -0,0 +1,382 @@ +import warnings + +import numpy as np + +import pandas as pd + + +class Methods: + params = ( + ["DataFrame", "Series"], + [("rolling", {"window": 10}), ("rolling", {"window": 1000}), ("expanding", {})], + ["int", "float"], + [ + "median", + "mean", + "max", + "min", + "std", + "count", + "skew", + "kurt", + "sum", + "sem", + "nunique", + ], + ) + param_names = ["constructor", "window_kwargs", "dtype", "method"] + + def setup(self, constructor, window_kwargs, dtype, method): + N = 10**5 + window, kwargs = window_kwargs + arr = (100 * np.random.random(N)).astype(dtype) + obj = getattr(pd, constructor)(arr) + self.window = getattr(obj, window)(**kwargs) + + def time_method(self, constructor, window_kwargs, dtype, method): + getattr(self.window, method)() + + def peakmem_method(self, constructor, window_kwargs, dtype, method): + getattr(self.window, method)() + + +class Apply: + params = ( + ["DataFrame", "Series"], + [3, 300], + ["int", "float"], + [sum, np.sum, lambda x: np.sum(x) + 5], + [True, False], + ) + param_names = ["constructor", "window", "dtype", "function", "raw"] + + def setup(self, constructor, window, dtype, function, raw): + N = 10**3 + arr = (100 * np.random.random(N)).astype(dtype) + self.roll = getattr(pd, constructor)(arr).rolling(window) + + def time_rolling(self, constructor, window, dtype, function, raw): + self.roll.apply(function, raw=raw) + + +class NumbaEngineMethods: + params = ( + ["DataFrame", "Series"], + ["int", "float"], + [("rolling", {"window": 10}), ("expanding", {})], + ["sum", "max", "min", "median", "mean", "var", "std"], + [True, False], + [None, 100], + ) + param_names = [ + "constructor", + "dtype", + "window_kwargs", + "method", + "parallel", + "cols", + ] + + def setup(self, constructor, dtype, window_kwargs, method, parallel, cols): + N = 10**3 + window, kwargs = window_kwargs + shape = (N, cols) if cols is not None and constructor != "Series" else N + arr = (100 * np.random.random(shape)).astype(dtype) + data = getattr(pd, constructor)(arr) + + # Warm the cache + with warnings.catch_warnings(record=True): + # Catch parallel=True not being applicable e.g. 1D data + self.window = getattr(data, window)(**kwargs) + getattr(self.window, method)( + engine="numba", engine_kwargs={"parallel": parallel} + ) + + def test_method(self, constructor, dtype, window_kwargs, method, parallel, cols): + with warnings.catch_warnings(record=True): + getattr(self.window, method)( + engine="numba", engine_kwargs={"parallel": parallel} + ) + + +class NumbaEngineApply: + params = ( + ["DataFrame", "Series"], + ["int", "float"], + [("rolling", {"window": 10}), ("expanding", {})], + [np.sum, lambda x: np.sum(x) + 5], + [True, False], + [None, 100], + ) + param_names = [ + "constructor", + "dtype", + "window_kwargs", + "function", + "parallel", + "cols", + ] + + def setup(self, constructor, dtype, window_kwargs, function, parallel, cols): + N = 10**3 + window, kwargs = window_kwargs + shape = (N, cols) if cols is not None and constructor != "Series" else N + arr = (100 * np.random.random(shape)).astype(dtype) + data = getattr(pd, constructor)(arr) + + # Warm the cache + with warnings.catch_warnings(record=True): + # Catch parallel=True not being applicable e.g. 1D data + self.window = getattr(data, window)(**kwargs) + self.window.apply( + function, raw=True, engine="numba", engine_kwargs={"parallel": parallel} + ) + + def test_method(self, constructor, dtype, window_kwargs, function, parallel, cols): + with warnings.catch_warnings(record=True): + self.window.apply( + function, raw=True, engine="numba", engine_kwargs={"parallel": parallel} + ) + + +class EWMMethods: + params = ( + ["DataFrame", "Series"], + [ + ({"halflife": 10}, "mean"), + ({"halflife": 10}, "std"), + ({"halflife": 1000}, "mean"), + ({"halflife": 1000}, "std"), + ( + { + "halflife": "1 Day", + "times": pd.date_range("1900", periods=10**5, freq="23s"), + }, + "mean", + ), + ], + ["int", "float"], + ) + param_names = ["constructor", "kwargs_method", "dtype"] + + def setup(self, constructor, kwargs_method, dtype): + N = 10**5 + kwargs, method = kwargs_method + arr = (100 * np.random.random(N)).astype(dtype) + self.method = method + self.ewm = getattr(pd, constructor)(arr).ewm(**kwargs) + + def time_ewm(self, constructor, kwargs_method, dtype): + getattr(self.ewm, self.method)() + + +class VariableWindowMethods(Methods): + params = ( + ["DataFrame", "Series"], + ["50s", "1h", "1d"], + ["int", "float"], + ["median", "mean", "max", "min", "std", "count", "skew", "kurt", "sum", "sem"], + ) + param_names = ["constructor", "window", "dtype", "method"] + + def setup(self, constructor, window, dtype, method): + N = 10**5 + arr = (100 * np.random.random(N)).astype(dtype) + index = pd.date_range("2017-01-01", periods=N, freq="5s") + self.window = getattr(pd, constructor)(arr, index=index).rolling(window) + + +class Pairwise: + params = ( + [({"window": 10}, "rolling"), ({"window": 1000}, "rolling"), ({}, "expanding")], + ["corr", "cov"], + [True, False], + ) + param_names = ["window_kwargs", "method", "pairwise"] + + def setup(self, kwargs_window, method, pairwise): + N = 10**4 + n_groups = 20 + kwargs, window = kwargs_window + groups = [i for _ in range(N // n_groups) for i in range(n_groups)] + arr = np.random.random(N) + self.df = pd.DataFrame(arr) + self.window = getattr(self.df, window)(**kwargs) + self.window_group = getattr( + pd.DataFrame({"A": groups, "B": arr}).groupby("A"), window + )(**kwargs) + + def time_pairwise(self, kwargs_window, method, pairwise): + getattr(self.window, method)(self.df, pairwise=pairwise) + + def time_groupby(self, kwargs_window, method, pairwise): + getattr(self.window_group, method)(self.df, pairwise=pairwise) + + +class Quantile: + params = ( + ["DataFrame", "Series"], + [10, 1000], + ["int", "float"], + [0, 0.5, 1], + ["linear", "nearest", "lower", "higher", "midpoint"], + ) + param_names = ["constructor", "window", "dtype", "percentile"] + + def setup(self, constructor, window, dtype, percentile, interpolation): + N = 10**5 + arr = np.random.random(N).astype(dtype) + self.roll = getattr(pd, constructor)(arr).rolling(window) + + def time_quantile(self, constructor, window, dtype, percentile, interpolation): + self.roll.quantile(percentile, interpolation=interpolation) + + +class Rank: + params = ( + ["DataFrame", "Series"], + [10, 1000], + ["int", "float"], + [True, False], + [True, False], + ["min", "max", "average"], + ) + param_names = [ + "constructor", + "window", + "dtype", + "percentile", + "ascending", + "method", + ] + + def setup(self, constructor, window, dtype, percentile, ascending, method): + N = 10**5 + arr = np.random.random(N).astype(dtype) + self.roll = getattr(pd, constructor)(arr).rolling(window) + + def time_rank(self, constructor, window, dtype, percentile, ascending, method): + self.roll.rank(pct=percentile, ascending=ascending, method=method) + + +class PeakMemFixedWindowMinMax: + params = ["min", "max"] + + def setup(self, operation): + N = 10**6 + arr = np.random.random(N) + self.roll = pd.Series(arr).rolling(2) + + def peakmem_fixed(self, operation): + for x in range(5): + getattr(self.roll, operation)() + + +class ForwardWindowMethods: + params = ( + ["DataFrame", "Series"], + [10, 1000], + ["int", "float"], + ["median", "mean", "max", "min", "kurt", "sum"], + ) + param_names = ["constructor", "window_size", "dtype", "method"] + + def setup(self, constructor, window_size, dtype, method): + N = 10**5 + arr = np.random.random(N).astype(dtype) + indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=window_size) + self.roll = getattr(pd, constructor)(arr).rolling(window=indexer) + + def time_rolling(self, constructor, window_size, dtype, method): + getattr(self.roll, method)() + + def peakmem_rolling(self, constructor, window_size, dtype, method): + getattr(self.roll, method)() + + +class Groupby: + params = ( + ["sum", "median", "mean", "max", "min", "kurt", "sum"], + [ + ("rolling", {"window": 2}), + ("rolling", {"window": "30s"}), + ("expanding", {}), + ], + ) + + def setup(self, method, window_kwargs): + N = 1000 + window, kwargs = window_kwargs + df = pd.DataFrame( + { + "A": [str(i) for i in range(N)] * 10, + "B": list(range(N)) * 10, + } + ) + if isinstance(kwargs.get("window", None), str): + df.index = pd.date_range(start="1900-01-01", freq="1min", periods=N * 10) + self.groupby_window = getattr(df.groupby("A"), window)(**kwargs) + + def time_method(self, method, window_kwargs): + getattr(self.groupby_window, method)() + + +class GroupbyLargeGroups: + # https://github.com/pandas-dev/pandas/issues/38038 + # specific example where the rolling operation on a larger dataframe + # is relatively cheap (few but large groups), but creation of + # MultiIndex of result can be expensive + + def setup(self): + N = 100000 + self.df = pd.DataFrame({"A": [1, 2] * (N // 2), "B": np.random.randn(N)}) + + def time_rolling_multiindex_creation(self): + self.df.groupby("A").rolling(3).mean() + + +class GroupbyEWM: + params = ["var", "std", "cov", "corr"] + param_names = ["method"] + + def setup(self, method): + df = pd.DataFrame({"A": range(50), "B": range(50)}) + self.gb_ewm = df.groupby("A").ewm(com=1.0) + + def time_groupby_method(self, method): + getattr(self.gb_ewm, method)() + + +class GroupbyEWMEngine: + params = ["cython", "numba"] + param_names = ["engine"] + + def setup(self, engine): + df = pd.DataFrame({"A": range(50), "B": range(50)}) + self.gb_ewm = df.groupby("A").ewm(com=1.0) + + def time_groupby_mean(self, engine): + self.gb_ewm.mean(engine=engine) + + +def table_method_func(x): + return np.sum(x, axis=0) + 1 + + +class TableMethod: + params = ["single", "table"] + param_names = ["method"] + + def setup(self, method): + self.df = pd.DataFrame(np.random.randn(10, 1000)) + + def time_apply(self, method): + self.df.rolling(2, method=method).apply( + table_method_func, raw=True, engine="numba" + ) + + def time_ewm_mean(self, method): + self.df.ewm(1, method=method).mean(engine="numba") + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/series_methods.py b/asv_bench/benchmarks/series_methods.py new file mode 100644 index 0000000000000..85d34cac5a7bf --- /dev/null +++ b/asv_bench/benchmarks/series_methods.py @@ -0,0 +1,425 @@ +from datetime import datetime + +import numpy as np + +from pandas import ( + NA, + Index, + NaT, + Series, + date_range, +) + + +class SeriesConstructor: + def setup(self): + self.idx = date_range( + start=datetime(2015, 10, 26), end=datetime(2016, 1, 1), freq="50s" + ) + self.data = dict(zip(self.idx, range(len(self.idx)))) + self.array = np.array([1, 2, 3]) + self.idx2 = Index(["a", "b", "c"]) + + def time_constructor_dict(self): + Series(data=self.data, index=self.idx) + + def time_constructor_no_data(self): + Series(data=None, index=self.idx) + + +class ToFrame: + params = [["int64", "datetime64[ns]", "category", "Int64"], [None, "foo"]] + param_names = ["dtype", "name"] + + def setup(self, dtype, name): + arr = np.arange(10**5) + ser = Series(arr, dtype=dtype) + self.ser = ser + + def time_to_frame(self, dtype, name): + self.ser.to_frame(name) + + +class NSort: + params = ["first", "last", "all"] + param_names = ["keep"] + + def setup(self, keep): + self.s = Series(np.random.randint(1, 10, 100000)) + + def time_nlargest(self, keep): + self.s.nlargest(3, keep=keep) + + def time_nsmallest(self, keep): + self.s.nsmallest(3, keep=keep) + + +class Dropna: + params = ["int", "datetime"] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10**6 + data = { + "int": np.random.randint(1, 10, N), + "datetime": date_range("2000-01-01", freq="s", periods=N), + } + self.s = Series(data[dtype]) + if dtype == "datetime": + self.s[np.random.randint(1, N, 100)] = NaT + + def time_dropna(self, dtype): + self.s.dropna() + + +class Fillna: + params = [ + [ + "datetime64[ns]", + "float32", + "float64", + "Float64", + "Int64", + "int64[pyarrow]", + "string", + "string[pyarrow]", + ], + ] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10**6 + if dtype == "datetime64[ns]": + data = date_range("2000-01-01", freq="s", periods=N) + na_value = NaT + elif dtype in ("float64", "Float64"): + data = np.random.randn(N) + na_value = np.nan + elif dtype in ("Int64", "int64[pyarrow]"): + data = np.arange(N) + na_value = NA + elif dtype in ("string", "string[pyarrow]"): + data = np.array([str(i) * 5 for i in range(N)], dtype=object) + na_value = NA + else: + raise NotImplementedError + fill_value = data[0] + ser = Series(data, dtype=dtype) + ser[::2] = na_value + self.ser = ser + self.fill_value = fill_value + + def time_fillna(self, dtype): + self.ser.fillna(value=self.fill_value) + + def time_ffill(self, dtype): + self.ser.ffill() + + def time_bfill(self, dtype): + self.ser.bfill() + + +class SearchSorted: + goal_time = 0.2 + params = [ + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "float16", + "float32", + "float64", + "str", + ] + param_names = ["dtype"] + + def setup(self, dtype): + N = 10**5 + data = np.array([1] * N + [2] * N + [3] * N).astype(dtype) + self.s = Series(data) + + def time_searchsorted(self, dtype): + key = "2" if dtype == "str" else 2 + self.s.searchsorted(key) + + +class Map: + params = ( + ["dict", "Series", "lambda"], + ["object", "category", "int"], + [None, "ignore"], + ) + param_names = ["mapper", "dtype", "na_action"] + + def setup(self, mapper, dtype, na_action): + map_size = 1000 + map_data = Series(map_size - np.arange(map_size), dtype=dtype) + + # construct mapper + if mapper == "Series": + self.map_data = map_data + elif mapper == "dict": + self.map_data = map_data.to_dict() + elif mapper == "lambda": + map_dict = map_data.to_dict() + self.map_data = lambda x: map_dict[x] + else: + raise NotImplementedError + + self.s = Series(np.random.randint(0, map_size, 10000), dtype=dtype) + + def time_map(self, mapper, dtype, na_action): + self.s.map(self.map_data, na_action=na_action) + + +class Clip: + params = [50, 1000, 10**5] + param_names = ["n"] + + def setup(self, n): + self.s = Series(np.random.randn(n)) + + def time_clip(self, n): + self.s.clip(0, 1) + + +class ClipDt: + def setup(self): + dr = date_range("20220101", periods=100_000, freq="s", tz="UTC") + self.clipper_dt = dr[0:1_000].repeat(100) + self.s = Series(dr) + + def time_clip(self): + self.s.clip(upper=self.clipper_dt) + + +class ValueCounts: + params = [[10**3, 10**4, 10**5], ["int", "uint", "float", "object"]] + param_names = ["N", "dtype"] + + def setup(self, N, dtype): + self.s = Series(np.random.randint(0, N, size=10 * N)).astype(dtype) + + def time_value_counts(self, N, dtype): + self.s.value_counts() + + +class ValueCountsEA: + params = [[10**3, 10**4, 10**5], [True, False]] + param_names = ["N", "dropna"] + + def setup(self, N, dropna): + self.s = Series(np.random.randint(0, N, size=10 * N), dtype="Int64") + self.s.loc[1] = NA + + def time_value_counts(self, N, dropna): + self.s.value_counts(dropna=dropna) + + +class ValueCountsObjectDropNAFalse: + params = [10**3, 10**4, 10**5] + param_names = ["N"] + + def setup(self, N): + self.s = Series(np.random.randint(0, N, size=10 * N)).astype("object") + + def time_value_counts(self, N): + self.s.value_counts(dropna=False) + + +class Mode: + params = [[10**3, 10**4, 10**5], ["int", "uint", "float", "object"]] + param_names = ["N", "dtype"] + + def setup(self, N, dtype): + self.s = Series(np.random.randint(0, N, size=10 * N)).astype(dtype) + + def time_mode(self, N, dtype): + self.s.mode() + + +class ModeObjectDropNAFalse: + params = [10**3, 10**4, 10**5] + param_names = ["N"] + + def setup(self, N): + self.s = Series(np.random.randint(0, N, size=10 * N)).astype("object") + + def time_mode(self, N): + self.s.mode(dropna=False) + + +class Dir: + def setup(self): + self.s = Series(index=Index([f"i-{i}" for i in range(10000)], dtype=object)) + + def time_dir_strings(self): + dir(self.s) + + +class SeriesGetattr: + # https://github.com/pandas-dev/pandas/issues/19764 + def setup(self): + self.s = Series(1, index=date_range("2012-01-01", freq="s", periods=10**6)) + + def time_series_datetimeindex_repr(self): + getattr(self.s, "a", None) + + +class All: + params = [[10**3, 10**6], ["fast", "slow"], ["bool", "boolean"]] + param_names = ["N", "case", "dtype"] + + def setup(self, N, case, dtype): + val = case != "fast" + self.s = Series([val] * N, dtype=dtype) + + def time_all(self, N, case, dtype): + self.s.all() + + +class Any: + params = [[10**3, 10**6], ["fast", "slow"], ["bool", "boolean"]] + param_names = ["N", "case", "dtype"] + + def setup(self, N, case, dtype): + val = case == "fast" + self.s = Series([val] * N, dtype=dtype) + + def time_any(self, N, case, dtype): + self.s.any() + + +class NanOps: + params = [ + [ + "var", + "mean", + "median", + "max", + "min", + "sum", + "std", + "sem", + "argmax", + "skew", + "kurt", + "prod", + ], + [10**3, 10**6], + ["int8", "int32", "int64", "float64", "Int64", "boolean"], + ] + param_names = ["func", "N", "dtype"] + + def setup(self, func, N, dtype): + if func == "argmax" and dtype in {"Int64", "boolean"}: + # Skip argmax for nullable int since this doesn't work yet (GH-24382) + raise NotImplementedError + self.s = Series(np.ones(N), dtype=dtype) + self.func = getattr(self.s, func) + + def time_func(self, func, N, dtype): + self.func() + + +class Rank: + param_names = ["dtype"] + params = [ + ["int", "uint", "float", "object"], + ] + + def setup(self, dtype): + self.s = Series(np.random.randint(0, 1000, size=100000), dtype=dtype) + + def time_rank(self, dtype): + self.s.rank() + + +class Iter: + param_names = ["dtype"] + params = [ + "bool", + "boolean", + "int64", + "Int64", + "float64", + "Float64", + "datetime64[ns]", + ] + + def setup(self, dtype): + N = 10**5 + if dtype in ["bool", "boolean"]: + data = np.repeat([True, False], N // 2) + elif dtype in ["int64", "Int64"]: + data = np.arange(N) + elif dtype in ["float64", "Float64"]: + data = np.random.randn(N) + elif dtype == "datetime64[ns]": + data = date_range("2000-01-01", freq="s", periods=N) + else: + raise NotImplementedError + + self.s = Series(data, dtype=dtype) + + def time_iter(self, dtype): + for v in self.s: + pass + + +class ToNumpy: + def setup(self): + N = 1_000_000 + self.ser = Series( + np.random.randn( + N, + ) + ) + + def time_to_numpy(self): + self.ser.to_numpy() + + def time_to_numpy_double_copy(self): + self.ser.to_numpy(dtype="float64", copy=True) + + def time_to_numpy_copy(self): + self.ser.to_numpy(copy=True) + + def time_to_numpy_float_with_nan(self): + self.ser.to_numpy(dtype="float64", na_value=np.nan) + + +class Replace: + param_names = ["num_to_replace"] + params = [100, 1000] + + def setup(self, num_to_replace): + N = 1_000_000 + self.arr = np.random.randn(N) + self.arr1 = self.arr.copy() + np.random.shuffle(self.arr1) + self.ser = Series(self.arr) + + self.to_replace_list = np.random.choice(self.arr, num_to_replace) + self.values_list = np.random.choice(self.arr1, num_to_replace) + + self.replace_dict = dict(zip(self.to_replace_list, self.values_list)) + + def time_replace_dict(self, num_to_replace): + self.ser.replace(self.replace_dict) + + def peakmem_replace_dict(self, num_to_replace): + self.ser.replace(self.replace_dict) + + def time_replace_list(self, num_to_replace): + self.ser.replace(self.to_replace_list, self.values_list) + + def peakmem_replace_list(self, num_to_replace): + self.ser.replace(self.to_replace_list, self.values_list) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/sparse.py b/asv_bench/benchmarks/sparse.py new file mode 100644 index 0000000000000..22a5511e4c678 --- /dev/null +++ b/asv_bench/benchmarks/sparse.py @@ -0,0 +1,227 @@ +import numpy as np +import scipy.sparse + +import pandas as pd +from pandas import ( + MultiIndex, + Series, + date_range, +) +from pandas.arrays import SparseArray + + +def make_array(size, dense_proportion, fill_value, dtype): + dense_size = int(size * dense_proportion) + arr = np.full(size, fill_value, dtype) + indexer = np.random.choice(np.arange(size), dense_size, replace=False) + arr[indexer] = np.random.choice(np.arange(100, dtype=dtype), dense_size) + return arr + + +class SparseSeriesToFrame: + def setup(self): + K = 50 + N = 50001 + rng = date_range("1/1/2000", periods=N, freq="min") + self.series = {} + for i in range(1, K): + data = np.random.randn(N)[:-i] + idx = rng[:-i] + data[100:] = np.nan + self.series[i] = Series(SparseArray(data), index=idx) + + def time_series_to_frame(self): + pd.DataFrame(self.series) + + +class SparseArrayConstructor: + params = ([0.1, 0.01], [0, np.nan], [np.int64, np.float64, object]) + param_names = ["dense_proportion", "fill_value", "dtype"] + + def setup(self, dense_proportion, fill_value, dtype): + N = 10**6 + self.array = make_array(N, dense_proportion, fill_value, dtype) + + def time_sparse_array(self, dense_proportion, fill_value, dtype): + SparseArray(self.array, fill_value=fill_value, dtype=dtype) + + +class SparseDataFrameConstructor: + def setup(self): + N = 1000 + self.sparse = scipy.sparse.rand(N, N, 0.005) + + def time_from_scipy(self): + pd.DataFrame.sparse.from_spmatrix(self.sparse) + + +class FromCoo: + def setup(self): + self.matrix = scipy.sparse.coo_matrix( + ([3.0, 1.0, 2.0], ([1, 0, 0], [0, 2, 3])), shape=(100, 100) + ) + + def time_sparse_series_from_coo(self): + Series.sparse.from_coo(self.matrix) + + +class ToCoo: + params = [True, False] + param_names = ["sort_labels"] + + def setup(self, sort_labels): + s = Series([np.nan] * 10000) + s[0] = 3.0 + s[100] = -1.0 + s[999] = 12.1 + + s_mult_lvl = s.set_axis(MultiIndex.from_product([range(10)] * 4)) + self.ss_mult_lvl = s_mult_lvl.astype("Sparse") + + s_two_lvl = s.set_axis(MultiIndex.from_product([range(100)] * 2)) + self.ss_two_lvl = s_two_lvl.astype("Sparse") + + def time_sparse_series_to_coo(self, sort_labels): + self.ss_mult_lvl.sparse.to_coo( + row_levels=[0, 1], column_levels=[2, 3], sort_labels=sort_labels + ) + + def time_sparse_series_to_coo_single_level(self, sort_labels): + self.ss_two_lvl.sparse.to_coo(sort_labels=sort_labels) + + +class ToCooFrame: + def setup(self): + N = 10000 + k = 10 + arr = np.zeros((N, k), dtype=float) + arr[0, 0] = 3.0 + arr[12, 7] = -1.0 + arr[0, 9] = 11.2 + self.df = pd.DataFrame(arr, dtype=pd.SparseDtype("float", fill_value=0.0)) + + def time_to_coo(self): + self.df.sparse.to_coo() + + +class Arithmetic: + params = ([0.1, 0.01], [0, np.nan]) + param_names = ["dense_proportion", "fill_value"] + + def setup(self, dense_proportion, fill_value): + N = 10**6 + arr1 = make_array(N, dense_proportion, fill_value, np.int64) + self.array1 = SparseArray(arr1, fill_value=fill_value) + arr2 = make_array(N, dense_proportion, fill_value, np.int64) + self.array2 = SparseArray(arr2, fill_value=fill_value) + + def time_make_union(self, dense_proportion, fill_value): + self.array1.sp_index.make_union(self.array2.sp_index) + + def time_intersect(self, dense_proportion, fill_value): + self.array1.sp_index.intersect(self.array2.sp_index) + + def time_add(self, dense_proportion, fill_value): + self.array1 + self.array2 + + def time_divide(self, dense_proportion, fill_value): + self.array1 / self.array2 + + +class ArithmeticBlock: + params = [np.nan, 0] + param_names = ["fill_value"] + + def setup(self, fill_value): + N = 10**6 + self.arr1 = self.make_block_array( + length=N, num_blocks=1000, block_size=10, fill_value=fill_value + ) + self.arr2 = self.make_block_array( + length=N, num_blocks=1000, block_size=10, fill_value=fill_value + ) + + def make_block_array(self, length, num_blocks, block_size, fill_value): + arr = np.full(length, fill_value) + indices = np.random.choice( + np.arange(0, length, block_size), num_blocks, replace=False + ) + for ind in indices: + arr[ind : ind + block_size] = np.random.randint(0, 100, block_size) + return SparseArray(arr, fill_value=fill_value) + + def time_make_union(self, fill_value): + self.arr1.sp_index.make_union(self.arr2.sp_index) + + def time_intersect(self, fill_value): + self.arr2.sp_index.intersect(self.arr2.sp_index) + + def time_addition(self, fill_value): + self.arr1 + self.arr2 + + def time_division(self, fill_value): + self.arr1 / self.arr2 + + +class MinMax: + params = (["min", "max"], [0.0, np.nan]) + param_names = ["func", "fill_value"] + + def setup(self, func, fill_value): + N = 1_000_000 + arr = make_array(N, 1e-5, fill_value, np.float64) + self.sp_arr = SparseArray(arr, fill_value=fill_value) + + def time_min_max(self, func, fill_value): + getattr(self.sp_arr, func)() + + +class Take: + params = ([np.array([0]), np.arange(100_000), np.full(100_000, -1)], [True, False]) + param_names = ["indices", "allow_fill"] + + def setup(self, indices, allow_fill): + N = 1_000_000 + fill_value = 0.0 + arr = make_array(N, 1e-5, fill_value, np.float64) + self.sp_arr = SparseArray(arr, fill_value=fill_value) + + def time_take(self, indices, allow_fill): + self.sp_arr.take(indices, allow_fill=allow_fill) + + +class GetItem: + def setup(self): + N = 1_000_000 + d = 1e-5 + arr = make_array(N, d, np.nan, np.float64) + self.sp_arr = SparseArray(arr) + + def time_integer_indexing(self): + self.sp_arr[78] + + def time_slice(self): + self.sp_arr[1:] + + +class GetItemMask: + params = [True, False, np.nan] + param_names = ["fill_value"] + + def setup(self, fill_value): + N = 1_000_000 + d = 1e-5 + arr = make_array(N, d, np.nan, np.float64) + self.sp_arr = SparseArray(arr) + b_arr = np.full(shape=N, fill_value=fill_value, dtype=np.bool_) + fv_inds = np.unique( + np.random.randint(low=0, high=N - 1, size=int(N * d), dtype=np.int32) + ) + b_arr[fv_inds] = True if pd.isna(fill_value) else not fill_value + self.sp_b_arr = SparseArray(b_arr, dtype=np.bool_, fill_value=fill_value) + + def time_mask(self, fill_value): + self.sp_arr[self.sp_b_arr] + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/stat_ops.py b/asv_bench/benchmarks/stat_ops.py new file mode 100644 index 0000000000000..8913293dfa20e --- /dev/null +++ b/asv_bench/benchmarks/stat_ops.py @@ -0,0 +1,168 @@ +import numpy as np + +import pandas as pd + +ops = ["mean", "sum", "median", "std", "skew", "kurt", "prod", "sem", "var"] + + +class FrameOps: + params = [ops, ["float", "int", "Int64"], [0, 1, None]] + param_names = ["op", "dtype", "axis"] + + def setup(self, op, dtype, axis): + values = np.random.randn(100000, 4) + if dtype == "Int64": + values = values.astype(int) + df = pd.DataFrame(values).astype(dtype) + self.df_func = getattr(df, op) + + def time_op(self, op, dtype, axis): + self.df_func(axis=axis) + + +class FrameMixedDtypesOps: + params = [ops, [0, 1, None]] + param_names = ["op", "axis"] + + def setup(self, op, axis): + if op in ("sum", "skew", "kurt", "prod", "sem", "var") or ( + (op, axis) + in ( + ("mean", 1), + ("mean", None), + ("median", 1), + ("median", None), + ("std", 1), + ("std", None), + ) + ): + # Skipping cases where datetime aggregations are not implemented + raise NotImplementedError + + N = 1_000_000 + df = pd.DataFrame( + { + "f": np.random.normal(0.0, 1.0, N), + "i": np.random.randint(0, N, N), + "ts": pd.date_range(start="1/1/2000", periods=N, freq="h"), + } + ) + + self.df_func = getattr(df, op) + + def time_op(self, op, axis): + self.df_func(axis=axis) + + +class FrameMultiIndexOps: + params = [ops] + param_names = ["op"] + + def setup(self, op): + levels = [np.arange(10), np.arange(100), np.arange(100)] + codes = [ + np.arange(10).repeat(10000), + np.tile(np.arange(100).repeat(100), 10), + np.tile(np.tile(np.arange(100), 100), 10), + ] + index = pd.MultiIndex(levels=levels, codes=codes) + df = pd.DataFrame(np.random.randn(len(index), 4), index=index) + self.df_func = getattr(df, op) + + def time_op(self, op): + self.df_func() + + +class SeriesOps: + params = [ops, ["float", "int"]] + param_names = ["op", "dtype"] + + def setup(self, op, dtype): + s = pd.Series(np.random.randn(100000)).astype(dtype) + self.s_func = getattr(s, op) + + def time_op(self, op, dtype): + self.s_func() + + +class SeriesMultiIndexOps: + params = [ops] + param_names = ["op"] + + def setup(self, op): + levels = [np.arange(10), np.arange(100), np.arange(100)] + codes = [ + np.arange(10).repeat(10000), + np.tile(np.arange(100).repeat(100), 10), + np.tile(np.tile(np.arange(100), 100), 10), + ] + index = pd.MultiIndex(levels=levels, codes=codes) + s = pd.Series(np.random.randn(len(index)), index=index) + self.s_func = getattr(s, op) + + def time_op(self, op): + self.s_func() + + +class Rank: + params = [["DataFrame", "Series"], [True, False]] + param_names = ["constructor", "pct"] + + def setup(self, constructor, pct): + values = np.random.randn(10**5) + self.data = getattr(pd, constructor)(values) + + def time_rank(self, constructor, pct): + self.data.rank(pct=pct) + + def time_average_old(self, constructor, pct): + self.data.rank(pct=pct) / len(self.data) + + +class Correlation: + params = [["spearman", "kendall", "pearson"]] + param_names = ["method"] + + def setup(self, method): + self.df = pd.DataFrame(np.random.randn(500, 15)) + self.df2 = pd.DataFrame(np.random.randn(500, 15)) + self.df_wide = pd.DataFrame(np.random.randn(500, 100)) + self.df_wide_nans = self.df_wide.where(np.random.random((500, 100)) < 0.9) + self.s = pd.Series(np.random.randn(500)) + self.s2 = pd.Series(np.random.randn(500)) + + def time_corr(self, method): + self.df.corr(method=method) + + def time_corr_wide(self, method): + self.df_wide.corr(method=method) + + def time_corr_wide_nans(self, method): + self.df_wide_nans.corr(method=method) + + def peakmem_corr_wide(self, method): + self.df_wide.corr(method=method) + + def time_corr_series(self, method): + self.s.corr(self.s2, method=method) + + def time_corrwith_cols(self, method): + self.df.corrwith(self.df2, method=method) + + def time_corrwith_rows(self, method): + self.df.corrwith(self.df2, axis=1, method=method) + + +class Covariance: + params = [] + param_names = [] + + def setup(self): + self.s = pd.Series(np.random.randn(100000)) + self.s2 = pd.Series(np.random.randn(100000)) + + def time_cov_series(self): + self.s.cov(self.s2) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/strftime.py b/asv_bench/benchmarks/strftime.py new file mode 100644 index 0000000000000..5f2832e361eb5 --- /dev/null +++ b/asv_bench/benchmarks/strftime.py @@ -0,0 +1,109 @@ +import numpy as np + +import pandas as pd +from pandas import offsets + + +class DatetimeStrftime: + timeout = 1500 + params = [1000, 10000] + param_names = ["nobs"] + + def setup(self, nobs): + d = "2018-11-29" + dt = "2018-11-26 11:18:27.0" + self.data = pd.DataFrame( + { + "dt": [np.datetime64(dt)] * nobs, + "d": [np.datetime64(d)] * nobs, + "r": [np.random.uniform()] * nobs, + } + ) + + def time_frame_date_to_str(self, nobs): + self.data["d"].astype(str) + + def time_frame_date_formatting_default(self, nobs): + self.data["d"].dt.strftime(date_format=None) + + def time_frame_date_formatting_default_explicit(self, nobs): + self.data["d"].dt.strftime(date_format="%Y-%m-%d") + + def time_frame_date_formatting_custom(self, nobs): + self.data["d"].dt.strftime(date_format="%Y---%m---%d") + + def time_frame_datetime_to_str(self, nobs): + self.data["dt"].astype(str) + + def time_frame_datetime_formatting_default(self, nobs): + self.data["dt"].dt.strftime(date_format=None) + + def time_frame_datetime_formatting_default_explicit_date_only(self, nobs): + self.data["dt"].dt.strftime(date_format="%Y-%m-%d") + + def time_frame_datetime_formatting_default_explicit(self, nobs): + self.data["dt"].dt.strftime(date_format="%Y-%m-%d %H:%M:%S") + + def time_frame_datetime_formatting_default_with_float(self, nobs): + self.data["dt"].dt.strftime(date_format="%Y-%m-%d %H:%M:%S.%f") + + def time_frame_datetime_formatting_custom(self, nobs): + self.data["dt"].dt.strftime(date_format="%Y-%m-%d --- %H:%M:%S") + + +class PeriodStrftime: + timeout = 1500 + params = ([1000, 10000], ["D", "h"]) + param_names = ["nobs", "freq"] + + def setup(self, nobs, freq): + self.data = pd.DataFrame( + { + "p": pd.period_range(start="2000-01-01", periods=nobs, freq=freq), + "r": [np.random.uniform()] * nobs, + } + ) + self.data["i"] = self.data["p"] + self.data.set_index("i", inplace=True) + if freq == "D": + self.default_fmt = "%Y-%m-%d" + elif freq == "h": + self.default_fmt = "%Y-%m-%d %H:00" + + def time_frame_period_to_str(self, nobs, freq): + self.data["p"].astype(str) + + def time_frame_period_formatting_default(self, nobs, freq): + self.data["p"].dt.strftime(date_format=None) + + def time_frame_period_formatting_default_explicit(self, nobs, freq): + self.data["p"].dt.strftime(date_format=self.default_fmt) + + def time_frame_period_formatting_custom(self, nobs, freq): + self.data["p"].dt.strftime(date_format="%Y-%m-%d --- %H:%M:%S") + + def time_frame_period_formatting_iso8601_strftime_Z(self, nobs, freq): + self.data["p"].dt.strftime(date_format="%Y-%m-%dT%H:%M:%SZ") + + def time_frame_period_formatting_iso8601_strftime_offset(self, nobs, freq): + """Not optimized yet as %z is not supported by `convert_strftime_format`""" + self.data["p"].dt.strftime(date_format="%Y-%m-%dT%H:%M:%S%z") + + +class BusinessHourStrftime: + timeout = 1500 + params = [1000, 10000] + param_names = ["nobs"] + + def setup(self, nobs): + self.data = pd.DataFrame( + { + "off": [offsets.BusinessHour()] * nobs, + } + ) + + def time_frame_offset_str(self, nobs): + self.data["off"].apply(str) + + def time_frame_offset_repr(self, nobs): + self.data["off"].apply(repr) diff --git a/asv_bench/benchmarks/strings.py b/asv_bench/benchmarks/strings.py new file mode 100644 index 0000000000000..467fab857d306 --- /dev/null +++ b/asv_bench/benchmarks/strings.py @@ -0,0 +1,299 @@ +import warnings + +import numpy as np + +from pandas import ( + NA, + Categorical, + DataFrame, + Index, + Series, +) +from pandas.arrays import StringArray + + +class Dtypes: + params = ["str", "string[python]", "string[pyarrow]"] + param_names = ["dtype"] + + def setup(self, dtype): + try: + self.s = Series( + Index([f"i-{i}" for i in range(10000)], dtype=object)._values, + dtype=dtype, + ) + except ImportError as err: + raise NotImplementedError from err + + +class Construction: + params = ( + ["series", "frame", "categorical_series"], + ["str", "string[python]", "string[pyarrow]"], + ) + param_names = ["pd_type", "dtype"] + pd_mapping = {"series": Series, "frame": DataFrame, "categorical_series": Series} + dtype_mapping = {"str": "str", "string[python]": object, "string[pyarrow]": object} + + def setup(self, pd_type, dtype): + series_arr = np.array( + [str(i) * 10 for i in range(100_000)], dtype=self.dtype_mapping[dtype] + ) + if pd_type == "series": + self.arr = series_arr + elif pd_type == "frame": + self.arr = series_arr.reshape((50_000, 2)).copy() + elif pd_type == "categorical_series": + # GH37371. Testing construction of string series/frames from ExtensionArrays + self.arr = Categorical(series_arr) + + def time_construction(self, pd_type, dtype): + self.pd_mapping[pd_type](self.arr, dtype=dtype) + + def peakmem_construction(self, pd_type, dtype): + self.pd_mapping[pd_type](self.arr, dtype=dtype) + + +class Methods(Dtypes): + def time_center(self, dtype): + self.s.str.center(100) + + def time_count(self, dtype): + self.s.str.count("A") + + def time_endswith(self, dtype): + self.s.str.endswith("A") + + def time_extract(self, dtype): + with warnings.catch_warnings(record=True): + self.s.str.extract("(\\w*)A(\\w*)") + + def time_findall(self, dtype): + self.s.str.findall("[A-Z]+") + + def time_find(self, dtype): + self.s.str.find("[A-Z]+") + + def time_rfind(self, dtype): + self.s.str.rfind("[A-Z]+") + + def time_fullmatch(self, dtype): + self.s.str.fullmatch("A") + + def time_get(self, dtype): + self.s.str.get(0) + + def time_len(self, dtype): + self.s.str.len() + + def time_join(self, dtype): + self.s.str.join(" ") + + def time_match(self, dtype): + self.s.str.match("A") + + def time_normalize(self, dtype): + self.s.str.normalize("NFC") + + def time_pad(self, dtype): + self.s.str.pad(100, side="both") + + def time_partition(self, dtype): + self.s.str.partition("A") + + def time_rpartition(self, dtype): + self.s.str.rpartition("A") + + def time_replace(self, dtype): + self.s.str.replace("A", "\x01\x01") + + def time_translate(self, dtype): + self.s.str.translate({"A": "\x01\x01"}) + + def time_slice(self, dtype): + self.s.str.slice(5, 15, 2) + + def time_startswith(self, dtype): + self.s.str.startswith("A") + + def time_strip(self, dtype): + self.s.str.strip("A") + + def time_rstrip(self, dtype): + self.s.str.rstrip("A") + + def time_lstrip(self, dtype): + self.s.str.lstrip("A") + + def time_title(self, dtype): + self.s.str.title() + + def time_upper(self, dtype): + self.s.str.upper() + + def time_lower(self, dtype): + self.s.str.lower() + + def time_wrap(self, dtype): + self.s.str.wrap(10) + + def time_zfill(self, dtype): + self.s.str.zfill(10) + + def time_isalnum(self, dtype): + self.s.str.isalnum() + + def time_isalpha(self, dtype): + self.s.str.isalpha() + + def time_isdecimal(self, dtype): + self.s.str.isdecimal() + + def time_isdigit(self, dtype): + self.s.str.isdigit() + + def time_islower(self, dtype): + self.s.str.islower() + + def time_isnumeric(self, dtype): + self.s.str.isnumeric() + + def time_isspace(self, dtype): + self.s.str.isspace() + + def time_istitle(self, dtype): + self.s.str.istitle() + + def time_isupper(self, dtype): + self.s.str.isupper() + + +class Repeat: + params = ["int", "array"] + param_names = ["repeats"] + + def setup(self, repeats): + N = 10**5 + self.s = Series(Index([f"i-{i}" for i in range(N)], dtype=object)) + repeat = {"int": 1, "array": np.random.randint(1, 3, N)} + self.values = repeat[repeats] + + def time_repeat(self, repeats): + self.s.str.repeat(self.values) + + +class Cat: + params = ([0, 3], [None, ","], [None, "-"], [0.0, 0.001, 0.15]) + param_names = ["other_cols", "sep", "na_rep", "na_frac"] + + def setup(self, other_cols, sep, na_rep, na_frac): + N = 10**5 + mask_gen = lambda: np.random.choice([True, False], N, p=[1 - na_frac, na_frac]) + self.s = Series(Index([f"i-{i}" for i in range(N)], dtype=object)).where( + mask_gen() + ) + if other_cols == 0: + # str.cat self-concatenates only for others=None + self.others = None + else: + self.others = DataFrame( + { + i: Index([f"i-{i}" for i in range(N)], dtype=object).where( + mask_gen() + ) + for i in range(other_cols) + } + ) + + def time_cat(self, other_cols, sep, na_rep, na_frac): + # before the concatenation (one caller + other_cols columns), the total + # expected fraction of rows containing any NaN is: + # reduce(lambda t, _: t + (1 - t) * na_frac, range(other_cols + 1), 0) + # for other_cols=3 and na_frac=0.15, this works out to ~48% + self.s.str.cat(others=self.others, sep=sep, na_rep=na_rep) + + +class Contains(Dtypes): + params = (Dtypes.params, [True, False]) + param_names = ["dtype", "regex"] + + def setup(self, dtype, regex): + super().setup(dtype) + + def time_contains(self, dtype, regex): + self.s.str.contains("A", regex=regex) + + +class Split(Dtypes): + params = (Dtypes.params, [True, False]) + param_names = ["dtype", "expand"] + + def setup(self, dtype, expand): + super().setup(dtype) + self.s = self.s.str.join("--") + + def time_split(self, dtype, expand): + self.s.str.split("--", expand=expand) + + def time_rsplit(self, dtype, expand): + self.s.str.rsplit("--", expand=expand) + + +class Extract(Dtypes): + params = (Dtypes.params, [True, False]) + param_names = ["dtype", "expand"] + + def setup(self, dtype, expand): + super().setup(dtype) + + def time_extract_single_group(self, dtype, expand): + with warnings.catch_warnings(record=True): + self.s.str.extract("(\\w*)A", expand=expand) + + +class Dummies(Dtypes): + def setup(self, dtype): + super().setup(dtype) + N = len(self.s) // 5 + self.s = self.s[:N].str.join("|") + + def time_get_dummies(self, dtype): + self.s.str.get_dummies("|") + + +class Encode: + def setup(self): + self.ser = Series(Index([f"i-{i}" for i in range(10_000)], dtype=object)) + + def time_encode_decode(self): + self.ser.str.encode("utf-8").str.decode("utf-8") + + +class Slice: + def setup(self): + self.s = Series(["abcdefg", np.nan] * 500000) + + def time_vector_slice(self): + # GH 2602 + self.s.str[:5] + + +class Iter(Dtypes): + def time_iter(self, dtype): + for i in self.s: + pass + + +class StringArrayConstruction: + def setup(self): + self.series_arr = np.array([str(i) * 10 for i in range(10**5)], dtype=object) + self.series_arr_nan = np.concatenate([self.series_arr, np.array([NA] * 1000)]) + + def time_string_array_construction(self): + StringArray(self.series_arr) + + def time_string_array_with_nan_construction(self): + StringArray(self.series_arr_nan) + + def peakmem_stringarray_construction(self): + StringArray(self.series_arr) diff --git a/asv_bench/benchmarks/timedelta.py b/asv_bench/benchmarks/timedelta.py new file mode 100644 index 0000000000000..cb0e4455e1a56 --- /dev/null +++ b/asv_bench/benchmarks/timedelta.py @@ -0,0 +1,61 @@ +""" +Timedelta benchmarks with non-tslibs dependencies. See +benchmarks.tslibs.timedelta for benchmarks that rely only on tslibs. +""" + +from pandas import ( + DataFrame, + Series, + timedelta_range, +) + + +class DatetimeAccessor: + def setup_cache(self): + N = 100000 + series = Series(timedelta_range("1 days", periods=N, freq="h")) + return series + + def time_dt_accessor(self, series): + series.dt + + def time_timedelta_days(self, series): + series.dt.days + + def time_timedelta_seconds(self, series): + series.dt.seconds + + def time_timedelta_microseconds(self, series): + series.dt.microseconds + + def time_timedelta_nanoseconds(self, series): + series.dt.nanoseconds + + +class TimedeltaIndexing: + def setup(self): + self.index = timedelta_range(start="1985", periods=1000, freq="D") + self.index2 = timedelta_range(start="1986", periods=1000, freq="D") + self.series = Series(range(1000), index=self.index) + self.timedelta = self.index[500] + + def time_get_loc(self): + self.index.get_loc(self.timedelta) + + def time_shallow_copy(self): + self.index._view() + + def time_series_loc(self): + self.series.loc[self.timedelta] + + def time_align(self): + DataFrame({"a": self.series, "b": self.series[:500]}) + + def time_intersection(self): + self.index.intersection(self.index2) + + def time_union(self): + self.index.union(self.index2) + + def time_unique(self): + self.index.unique() diff --git a/asv_bench/benchmarks/timeseries.py b/asv_bench/benchmarks/timeseries.py new file mode 100644 index 0000000000000..8deec502898d9 --- /dev/null +++ b/asv_bench/benchmarks/timeseries.py @@ -0,0 +1,297 @@ +from datetime import timedelta + +import dateutil +import numpy as np + +from pandas import ( + DataFrame, + Series, + date_range, + period_range, + timedelta_range, +) + +from pandas.tseries.frequencies import infer_freq + +try: + from pandas.plotting._matplotlib.converter import DatetimeConverter +except ImportError: + from pandas.tseries.converter import DatetimeConverter + + +class DatetimeIndex: + params = ["dst", "repeated", "tz_aware", "tz_local", "tz_naive"] + param_names = ["index_type"] + + def setup(self, index_type): + N = 100000 + dtidxes = { + "dst": date_range( + start="10/29/2000 1:00:00", end="10/29/2000 1:59:59", freq="s" + ), + "repeated": date_range(start="2000", periods=N // 10, freq="s").repeat(10), + "tz_aware": date_range(start="2000", periods=N, freq="s", tz="US/Eastern"), + "tz_local": date_range( + start="2000", periods=N, freq="s", tz=dateutil.tz.tzlocal() + ), + "tz_naive": date_range(start="2000", periods=N, freq="s"), + } + self.index = dtidxes[index_type] + + def time_add_timedelta(self, index_type): + self.index + timedelta(minutes=2) + + def time_normalize(self, index_type): + self.index.normalize() + + def time_unique(self, index_type): + self.index.unique() + + def time_to_time(self, index_type): + self.index.time + + def time_get(self, index_type): + self.index[0] + + def time_timeseries_is_month_start(self, index_type): + self.index.is_month_start + + def time_to_date(self, index_type): + self.index.date + + def time_to_pydatetime(self, index_type): + self.index.to_pydatetime() + + def time_is_dates_only(self, index_type): + self.index._is_dates_only + + +class TzLocalize: + params = [None, "US/Eastern", "UTC", dateutil.tz.tzutc()] + param_names = "tz" + + def setup(self, tz): + dst_rng = date_range( + start="10/29/2000 1:00:00", end="10/29/2000 1:59:59", freq="s" + ) + self.index = date_range(start="10/29/2000", end="10/29/2000 00:59:59", freq="s") + self.index = self.index.append(dst_rng) + self.index = self.index.append(dst_rng) + self.index = self.index.append( + date_range(start="10/29/2000 2:00:00", end="10/29/2000 3:00:00", freq="s") + ) + + def time_infer_dst(self, tz): + self.index.tz_localize(tz, ambiguous="infer") + + +class ResetIndex: + params = [None, "US/Eastern"] + param_names = "tz" + + def setup(self, tz): + idx = date_range(start="1/1/2000", periods=1000, freq="h", tz=tz) + self.df = DataFrame(np.random.randn(1000, 2), index=idx) + + def time_reset_datetimeindex(self, tz): + self.df.reset_index() + + +class InferFreq: + # This depends mostly on code in _libs/, tseries/, and core.algos.unique + params = [None, "D", "B"] + param_names = ["freq"] + + def setup(self, freq): + if freq is None: + self.idx = date_range(start="1/1/1700", freq="D", periods=10000) + self.idx._data._freq = None + else: + self.idx = date_range(start="1/1/1700", freq=freq, periods=10000) + + def time_infer_freq(self, freq): + infer_freq(self.idx) + + +class TimeDatetimeConverter: + def setup(self): + N = 100000 + self.rng = date_range(start="1/1/2000", periods=N, freq="min") + + def time_convert(self): + DatetimeConverter.convert(self.rng, None, None) + + +class Iteration: + params = [date_range, period_range, timedelta_range] + param_names = ["time_index"] + + def setup(self, time_index): + N = 10**6 + if time_index is timedelta_range: + self.idx = time_index(start=0, freq="min", periods=N) + else: + self.idx = time_index(start="20140101", freq="min", periods=N) + self.exit = 10000 + + def time_iter(self, time_index): + for _ in self.idx: + pass + + def time_iter_preexit(self, time_index): + for i, _ in enumerate(self.idx): + if i > self.exit: + break + + +class ResampleDataFrame: + params = ["max", "mean", "min"] + param_names = ["method"] + + def setup(self, method): + rng = date_range(start="20130101", periods=100000, freq="50ms") + df = DataFrame(np.random.randn(100000, 2), index=rng) + self.resample = getattr(df.resample("1s"), method) + + def time_method(self, method): + self.resample() + + +class ResampleSeries: + params = (["period", "datetime"], ["5min", "1D"], ["mean", "ohlc"]) + param_names = ["index", "freq", "method"] + + def setup(self, index, freq, method): + indexes = { + "period": period_range(start="1/1/2000", end="1/1/2001", freq="min"), + "datetime": date_range(start="1/1/2000", end="1/1/2001", freq="min"), + } + idx = indexes[index] + ts = Series(np.random.randn(len(idx)), index=idx) + self.resample = getattr(ts.resample(freq), method) + + def time_resample(self, index, freq, method): + self.resample() + + +class ResampleDatetetime64: + # GH 7754 + def setup(self): + rng3 = date_range( + start="2000-01-01 00:00:00", end="2000-01-01 10:00:00", freq="555000us" + ) + self.dt_ts = Series(5, rng3, dtype="datetime64[ns]") + + def time_resample(self): + self.dt_ts.resample("1s").last() + + +class AsOf: + params = ["DataFrame", "Series"] + param_names = ["constructor"] + + def setup(self, constructor): + N = 10000 + M = 10 + rng = date_range(start="1/1/1990", periods=N, freq="53s") + data = { + "DataFrame": DataFrame(np.random.randn(N, M)), + "Series": Series(np.random.randn(N)), + } + self.ts = data[constructor] + self.ts.index = rng + self.ts2 = self.ts.copy() + self.ts2.iloc[250:5000] = np.nan + self.ts3 = self.ts.copy() + self.ts3.iloc[-5000:] = np.nan + self.dates = date_range(start="1/1/1990", periods=N * 10, freq="5s") + self.date = self.dates[0] + self.date_last = self.dates[-1] + self.date_early = self.date - timedelta(10) + + # test speed of pre-computing NAs. + def time_asof(self, constructor): + self.ts.asof(self.dates) + + # should be roughly the same as above. + def time_asof_nan(self, constructor): + self.ts2.asof(self.dates) + + # test speed of the code path for a scalar index + # without *while* loop + def time_asof_single(self, constructor): + self.ts.asof(self.date) + + # test speed of the code path for a scalar index + # before the start. should be the same as above. + def time_asof_single_early(self, constructor): + self.ts.asof(self.date_early) + + # test the speed of the code path for a scalar index + # with a long *while* loop. should still be much + # faster than pre-computing all the NAs. + def time_asof_nan_single(self, constructor): + self.ts3.asof(self.date_last) + + +class SortIndex: + params = [True, False] + param_names = ["monotonic"] + + def setup(self, monotonic): + N = 10**5 + idx = date_range(start="1/1/2000", periods=N, freq="s") + self.s = Series(np.random.randn(N), index=idx) + if not monotonic: + self.s = self.s.sample(frac=1) + + def time_sort_index(self, monotonic): + self.s.sort_index() + + def time_get_slice(self, monotonic): + self.s[:10000] + + +class Lookup: + def setup(self): + N = 1500000 + rng = date_range(start="1/1/2000", periods=N, freq="s") + self.ts = Series(1, index=rng) + self.lookup_val = rng[N // 2] + + def time_lookup_and_cleanup(self): + self.ts[self.lookup_val] + self.ts.index._cleanup() + + +class DatetimeAccessor: + params = [None, "US/Eastern", "UTC", dateutil.tz.tzutc()] + param_names = "tz" + + def setup(self, tz): + N = 100000 + self.series = Series(date_range(start="1/1/2000", periods=N, freq="min", tz=tz)) + + def time_dt_accessor(self, tz): + self.series.dt + + def time_dt_accessor_normalize(self, tz): + self.series.dt.normalize() + + def time_dt_accessor_month_name(self, tz): + self.series.dt.month_name() + + def time_dt_accessor_day_name(self, tz): + self.series.dt.day_name() + + def time_dt_accessor_time(self, tz): + self.series.dt.time + + def time_dt_accessor_date(self, tz): + self.series.dt.date + + def time_dt_accessor_year(self, tz): + self.series.dt.year + + +from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/tslibs/__init__.py b/asv_bench/benchmarks/tslibs/__init__.py new file mode 100644 index 0000000000000..815cf55b291cd --- /dev/null +++ b/asv_bench/benchmarks/tslibs/__init__.py @@ -0,0 +1,7 @@ +""" +Benchmarks in this directory should depend only on tslibs, tseries.offsets, +and to_offset. + +i.e. any code changes that do not touch those files should not need to +run these benchmarks. +""" diff --git a/asv_bench/benchmarks/tslibs/fields.py b/asv_bench/benchmarks/tslibs/fields.py new file mode 100644 index 0000000000000..fe31879e67a67 --- /dev/null +++ b/asv_bench/benchmarks/tslibs/fields.py @@ -0,0 +1,82 @@ +import numpy as np + +from pandas._libs.tslibs.fields import ( + get_date_field, + get_start_end_field, + get_timedelta_field, +) + +from .tslib import _sizes + + +class TimeGetTimedeltaField: + params = [ + _sizes, + ["seconds", "microseconds", "nanoseconds"], + ] + param_names = ["size", "field"] + + def setup(self, size, field): + arr = np.random.randint(0, 10, size=size, dtype="i8") + self.i8data = arr + arr = np.random.randint(-86400 * 1_000_000_000, 0, size=size, dtype="i8") + self.i8data_negative = arr + + def time_get_timedelta_field(self, size, field): + get_timedelta_field(self.i8data, field) + + def time_get_timedelta_field_negative_td(self, size, field): + get_timedelta_field(self.i8data_negative, field) + + +class TimeGetDateField: + params = [ + _sizes, + [ + "Y", + "M", + "D", + "h", + "m", + "s", + "us", + "ns", + "doy", + "dow", + "woy", + "q", + "dim", + "is_leap_year", + ], + ] + param_names = ["size", "field"] + + def setup(self, size, field): + arr = np.random.randint(0, 10, size=size, dtype="i8") + self.i8data = arr + + def time_get_date_field(self, size, field): + get_date_field(self.i8data, field) + + +class TimeGetStartEndField: + params = [ + _sizes, + ["start", "end"], + ["month", "quarter", "year"], + ["B", None, "QS"], + [12, 3, 5], + ] + param_names = ["size", "side", "period", "freqstr", "month_kw"] + + def setup(self, size, side, period, freqstr, month_kw): + arr = np.random.randint(0, 10, size=size, dtype="i8") + self.i8data = arr + + self.attrname = f"is_{period}_{side}" + + def time_get_start_end_field(self, size, side, period, freqstr, month_kw): + get_start_end_field(self.i8data, self.attrname, freqstr, month_kw=month_kw) + + +from ..pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/asv_bench/benchmarks/tslibs/normalize.py b/asv_bench/benchmarks/tslibs/normalize.py new file mode 100644 index 0000000000000..b263ae21422b6 --- /dev/null +++ b/asv_bench/benchmarks/tslibs/normalize.py @@ -0,0 +1,45 @@ +try: + from pandas._libs.tslibs import ( + is_date_array_normalized, + normalize_i8_timestamps, + ) +except ImportError: + from pandas._libs.tslibs.conversion import ( + normalize_i8_timestamps, + is_date_array_normalized, + ) + +import pandas as pd + +from .tslib import ( + _sizes, + _tzs, + tzlocal_obj, +) + + +class Normalize: + params = [ + _sizes, + _tzs, + ] + param_names = ["size", "tz"] + + def setup(self, size, tz): + # use an array that will have is_date_array_normalized give True, + # so we do not short-circuit early. + dti = pd.date_range("2016-01-01", periods=10, tz=tz).repeat(size // 10) + self.i8data = dti.asi8 + + if size == 10**6 and tz is tzlocal_obj: + # tzlocal is cumbersomely slow, so skip to keep runtime in check + raise NotImplementedError + + def time_normalize_i8_timestamps(self, size, tz): + # 10 i.e. NPY_FR_ns + normalize_i8_timestamps(self.i8data, tz, 10) + + def time_is_date_array_normalized(self, size, tz): + # TODO: cases with different levels of short-circuiting + # 10 i.e. NPY_FR_ns + is_date_array_normalized(self.i8data, tz, 10) diff --git a/asv_bench/benchmarks/tslibs/offsets.py b/asv_bench/benchmarks/tslibs/offsets.py new file mode 100644 index 0000000000000..55bd3c31c055c --- /dev/null +++ b/asv_bench/benchmarks/tslibs/offsets.py @@ -0,0 +1,86 @@ +""" +offsets benchmarks that rely only on tslibs. See benchmarks.offset for +offsets benchmarks that rely on other parts of pandas. +""" + +from datetime import datetime + +import numpy as np + +from pandas import offsets + +try: + import pandas.tseries.holiday +except ImportError: + pass + +hcal = pandas.tseries.holiday.USFederalHolidayCalendar() +# These offsets currently raise a NotImplementedError with .apply_index() +non_apply = [ + offsets.Day(), + offsets.BYearEnd(), + offsets.BYearBegin(), + offsets.BQuarterEnd(), + offsets.BQuarterBegin(), + offsets.BMonthEnd(), + offsets.BMonthBegin(), + offsets.CustomBusinessDay(), + offsets.CustomBusinessDay(calendar=hcal), + offsets.CustomBusinessMonthBegin(calendar=hcal), + offsets.CustomBusinessMonthEnd(calendar=hcal), + offsets.CustomBusinessMonthEnd(calendar=hcal), +] +other_offsets = [ + offsets.YearEnd(), + offsets.YearBegin(), + offsets.QuarterEnd(), + offsets.QuarterBegin(), + offsets.MonthEnd(), + offsets.MonthBegin(), + offsets.DateOffset(months=2, days=2), + offsets.BusinessDay(), + offsets.SemiMonthEnd(), + offsets.SemiMonthBegin(), +] +offset_objs = non_apply + other_offsets + + +class OnOffset: + params = offset_objs + param_names = ["offset"] + + def setup(self, offset): + self.dates = [ + datetime(2016, m, d) + for m in [10, 11, 12] + for d in [1, 2, 3, 28, 29, 30, 31] + if not (m == 11 and d == 31) + ] + + def time_on_offset(self, offset): + for date in self.dates: + offset.is_on_offset(date) + + +class OffestDatetimeArithmetic: + params = offset_objs + param_names = ["offset"] + + def setup(self, offset): + self.date = datetime(2011, 1, 1) + self.dt64 = np.datetime64("2011-01-01 09:00Z") + + def time_add_np_dt64(self, offset): + offset + self.dt64 + + def time_add(self, offset): + self.date + offset + + def time_add_10(self, offset): + self.date + (10 * offset) + + def time_subtract(self, offset): + self.date - offset + + def time_subtract_10(self, offset): + self.date - (10 * offset) diff --git a/asv_bench/benchmarks/tslibs/period.py b/asv_bench/benchmarks/tslibs/period.py new file mode 100644 index 0000000000000..af3bfac6d3d01 --- /dev/null +++ b/asv_bench/benchmarks/tslibs/period.py @@ -0,0 +1,162 @@ +""" +Period benchmarks that rely only on tslibs. See benchmarks.period for +Period benchmarks that rely on other parts of pandas. +""" + +import numpy as np + +from pandas._libs.tslibs.period import ( + Period, + periodarr_to_dt64arr, +) + +from pandas.tseries.frequencies import to_offset + +from .tslib import ( + _sizes, + _tzs, + tzlocal_obj, +) + +try: + from pandas._libs.tslibs.vectorized import dt64arr_to_periodarr +except ImportError: + from pandas._libs.tslibs.period import dt64arr_to_periodarr + + +class PeriodProperties: + params = ( + ["M", "min"], + [ + "year", + "month", + "day", + "hour", + "minute", + "second", + "is_leap_year", + "quarter", + "qyear", + "week", + "daysinmonth", + "dayofweek", + "dayofyear", + "start_time", + "end_time", + ], + ) + param_names = ["freq", "attr"] + + def setup(self, freq, attr): + self.per = Period("2012-06-01", freq=freq) + + def time_property(self, freq, attr): + getattr(self.per, attr) + + +class PeriodUnaryMethods: + params = ["M", "min"] + param_names = ["freq"] + + def setup(self, freq): + self.per = Period("2012-06-01", freq=freq) + if freq == "M": + self.default_fmt = "%Y-%m" + elif freq == "min": + self.default_fmt = "%Y-%m-%d %H:%M" + + def time_to_timestamp(self, freq): + self.per.to_timestamp() + + def time_now(self, freq): + self.per.now(freq) + + def time_asfreq(self, freq): + self.per.asfreq("Y") + + def time_str(self, freq): + str(self.per) + + def time_repr(self, freq): + repr(self.per) + + def time_strftime_default(self, freq): + self.per.strftime(None) + + def time_strftime_default_explicit(self, freq): + self.per.strftime(self.default_fmt) + + def time_strftime_custom(self, freq): + self.per.strftime("%b. %d, %Y was a %A") + + +class PeriodConstructor: + params = [["D"], [True, False]] + param_names = ["freq", "is_offset"] + + def setup(self, freq, is_offset): + if is_offset: + self.freq = to_offset(freq) + else: + self.freq = freq + + def time_period_constructor(self, freq, is_offset): + Period("2012-06-01", freq=freq) + + +_freq_ints = [ + 1000, + 1011, # Annual - November End + 2000, + 2011, # Quarterly - November End + 3000, + 4000, + 4006, # Weekly - Saturday End + 5000, + 6000, + 7000, + 8000, + 9000, + 10000, + 11000, + 12000, +] + + +class TimePeriodArrToDT64Arr: + params = [ + _sizes, + _freq_ints, + ] + param_names = ["size", "freq"] + + def setup(self, size, freq): + arr = np.arange(10, dtype="i8").repeat(size // 10) + self.i8values = arr + + def time_periodarray_to_dt64arr(self, size, freq): + periodarr_to_dt64arr(self.i8values, freq) + + +class TimeDT64ArrToPeriodArr: + params = [ + _sizes, + _freq_ints, + _tzs, + ] + param_names = ["size", "freq", "tz"] + + def setup(self, size, freq, tz): + if size == 10**6 and tz is tzlocal_obj: + # tzlocal is cumbersomely slow, so skip to keep runtime in check + raise NotImplementedError + + # we pick 2**55 because smaller values end up returning + # -1 from npy_datetimestruct_to_datetime with NPY_FR_Y frequency + # this artificially slows down functions since -1 is also the + # error sentinel + arr = np.arange(2**55, 2**55 + 10, dtype="i8").repeat(size // 10) + self.i8values = arr + + def time_dt64arr_to_periodarr(self, size, freq, tz): + dt64arr_to_periodarr(self.i8values, freq, tz) diff --git a/asv_bench/benchmarks/tslibs/resolution.py b/asv_bench/benchmarks/tslibs/resolution.py new file mode 100644 index 0000000000000..6317d299379d3 --- /dev/null +++ b/asv_bench/benchmarks/tslibs/resolution.py @@ -0,0 +1,53 @@ +""" +ipython analogue: + +tr = TimeResolution() +mi = pd.MultiIndex.from_product(tr.params[:-1] + ([str(x) for x in tr.params[-1]],)) +df = pd.DataFrame(np.nan, index=mi, columns=["mean", "stdev"]) + +for unit in tr.params[0]: + for size in tr.params[1]: + for tz in tr.params[2]: + tr.setup(unit, size, tz) + key = (unit, size, str(tz)) + print(key) + + val = %timeit -o tr.time_get_resolution(unit, size, tz) + + df.loc[key] = (val.average, val.stdev) + +""" + +import numpy as np + +try: + from pandas._libs.tslibs import get_resolution +except ImportError: + from pandas._libs.tslibs.resolution import get_resolution + +from .tslib import ( + _sizes, + _tzs, + tzlocal_obj, +) + + +class TimeResolution: + params = ( + ["D", "h", "m", "s", "us", "ns"], + _sizes, + _tzs, + ) + param_names = ["unit", "size", "tz"] + + def setup(self, unit, size, tz): + if size == 10**6 and tz is tzlocal_obj: + # tzlocal is cumbersomely slow, so skip to keep runtime in check + raise NotImplementedError + + arr = np.random.randint(0, 10, size=size, dtype="i8") + arr = arr.view(f"M8[{unit}]").astype("M8[ns]").view("i8") + self.i8data = arr + + def time_get_resolution(self, unit, size, tz): + get_resolution(self.i8data, tz) diff --git a/asv_bench/benchmarks/tslibs/timedelta.py b/asv_bench/benchmarks/tslibs/timedelta.py new file mode 100644 index 0000000000000..9d9689fcfa94b --- /dev/null +++ b/asv_bench/benchmarks/tslibs/timedelta.py @@ -0,0 +1,70 @@ +""" +Timedelta benchmarks that rely only on tslibs. See benchmarks.timedeltas for +Timedelta benchmarks that rely on other parts of pandas. +""" + +import datetime + +import numpy as np + +from pandas import Timedelta + + +class TimedeltaConstructor: + def setup(self): + self.nptimedelta64 = np.timedelta64(3600) + self.dttimedelta = datetime.timedelta(seconds=3600) + self.td = Timedelta(3600, unit="s") + + def time_from_int(self): + Timedelta(123456789) + + def time_from_unit(self): + Timedelta(1, unit="D") + + def time_from_components(self): + Timedelta( + days=1, + hours=2, + minutes=3, + seconds=4, + milliseconds=5, + microseconds=6, + nanoseconds=7, + ) + + def time_from_datetime_timedelta(self): + Timedelta(self.dttimedelta) + + def time_from_np_timedelta(self): + Timedelta(self.nptimedelta64) + + def time_from_string(self): + Timedelta("1 days") + + def time_from_iso_format(self): + Timedelta("P4DT12H30M5S") + + def time_from_missing(self): + Timedelta("nat") + + def time_from_pd_timedelta(self): + Timedelta(self.td) + + +class TimedeltaProperties: + def setup_cache(self): + td = Timedelta(days=365, minutes=35, seconds=25, milliseconds=35) + return td + + def time_timedelta_days(self, td): + td.days + + def time_timedelta_seconds(self, td): + td.seconds + + def time_timedelta_microseconds(self, td): + td.microseconds + + def time_timedelta_nanoseconds(self, td): + td.nanoseconds diff --git a/asv_bench/benchmarks/tslibs/timestamp.py b/asv_bench/benchmarks/tslibs/timestamp.py new file mode 100644 index 0000000000000..6145966fb6a0e --- /dev/null +++ b/asv_bench/benchmarks/tslibs/timestamp.py @@ -0,0 +1,155 @@ +from datetime import ( + datetime, + timezone, +) +import zoneinfo + +import numpy as np + +from pandas import Timestamp + +from .tslib import _tzs + + +class TimestampConstruction: + def setup(self): + self.npdatetime64 = np.datetime64("2020-01-01 00:00:00") + self.dttime_unaware = datetime(2020, 1, 1, 0, 0, 0) + self.dttime_aware = datetime(2020, 1, 1, 0, 0, 0, 0, timezone.utc) + self.ts = Timestamp("2020-01-01 00:00:00") + + def time_parse_iso8601_no_tz(self): + Timestamp("2017-08-25 08:16:14") + + def time_parse_iso8601_tz(self): + Timestamp("2017-08-25 08:16:14-0500") + + def time_parse_dateutil(self): + Timestamp("2017/08/25 08:16:14 AM") + + def time_parse_today(self): + Timestamp("today") + + def time_parse_now(self): + Timestamp("now") + + def time_fromordinal(self): + Timestamp.fromordinal(730120) + + def time_fromtimestamp(self): + Timestamp.fromtimestamp(1515448538) + + def time_from_npdatetime64(self): + Timestamp(self.npdatetime64) + + def time_from_datetime_unaware(self): + Timestamp(self.dttime_unaware) + + def time_from_datetime_aware(self): + Timestamp(self.dttime_aware) + + def time_from_pd_timestamp(self): + Timestamp(self.ts) + + +class TimestampProperties: + params = [_tzs] + param_names = ["tz"] + + def setup(self, tz): + self.ts = Timestamp("2017-08-25 08:16:14", tzinfo=tz) + + def time_tz(self, tz): + self.ts.tz + + def time_dayofweek(self, tz): + self.ts.dayofweek + + def time_dayofyear(self, tz): + self.ts.dayofyear + + def time_week(self, tz): + self.ts.week + + def time_quarter(self, tz): + self.ts.quarter + + def time_days_in_month(self, tz): + self.ts.days_in_month + + def time_is_month_start(self, tz): + self.ts.is_month_start + + def time_is_month_end(self, tz): + self.ts.is_month_end + + def time_is_quarter_start(self, tz): + self.ts.is_quarter_start + + def time_is_quarter_end(self, tz): + self.ts.is_quarter_end + + def time_is_year_start(self, tz): + self.ts.is_year_start + + def time_is_year_end(self, tz): + self.ts.is_year_end + + def time_is_leap_year(self, tz): + self.ts.is_leap_year + + def time_microsecond(self, tz): + self.ts.microsecond + + def time_month_name(self, tz): + self.ts.month_name() + + def time_weekday_name(self, tz): + self.ts.day_name() + + +class TimestampOps: + params = _tzs + param_names = ["tz"] + + def setup(self, tz): + self.ts = Timestamp("2017-08-25 08:16:14", tz=tz) + + def time_replace_tz(self, tz): + self.ts.replace(tzinfo=zoneinfo.ZoneInfo("US/Eastern")) + + def time_replace_None(self, tz): + self.ts.replace(tzinfo=None) + + def time_to_pydatetime(self, tz): + self.ts.to_pydatetime() + + def time_normalize(self, tz): + self.ts.normalize() + + def time_tz_convert(self, tz): + if self.ts.tz is not None: + self.ts.tz_convert(tz) + + def time_tz_localize(self, tz): + if self.ts.tz is None: + self.ts.tz_localize(tz) + + def time_to_julian_date(self, tz): + self.ts.to_julian_date() + + def time_floor(self, tz): + self.ts.floor("5min") + + def time_ceil(self, tz): + self.ts.ceil("5min") + + +class TimestampAcrossDst: + def setup(self): + dt = datetime(2016, 3, 27, 1, fold=0) + self.tzinfo = dt.astimezone(zoneinfo.ZoneInfo("Europe/Berlin")).tzinfo + self.ts2 = Timestamp(dt) + + def time_replace_across_dst(self): + self.ts2.replace(tzinfo=self.tzinfo) diff --git a/asv_bench/benchmarks/tslibs/tslib.py b/asv_bench/benchmarks/tslibs/tslib.py new file mode 100644 index 0000000000000..885cf48d01743 --- /dev/null +++ b/asv_bench/benchmarks/tslibs/tslib.py @@ -0,0 +1,69 @@ +""" +ipython analogue: + +tr = TimeIntsToPydatetime() +mi = pd.MultiIndex.from_product( + tr.params[:-1] + ([str(x) for x in tr.params[-1]],) +) +df = pd.DataFrame(np.nan, index=mi, columns=["mean", "stdev"]) +for box in tr.params[0]: + for size in tr.params[1]: + for tz in tr.params[2]: + tr.setup(box, size, tz) + key = (box, size, str(tz)) + print(key) + val = %timeit -o tr.time_ints_to_pydatetime(box, size, tz) + df.loc[key] = (val.average, val.stdev) +""" + +from datetime import ( + timedelta, + timezone, +) +import zoneinfo + +from dateutil.tz import ( + gettz, + tzlocal, +) +import numpy as np + +try: + from pandas._libs.tslibs import ints_to_pydatetime +except ImportError: + from pandas._libs.tslib import ints_to_pydatetime + +tzlocal_obj = tzlocal() +_tzs = [ + None, + timezone.utc, + timezone(timedelta(minutes=60)), + zoneinfo.ZoneInfo("US/Pacific"), + gettz("Asia/Tokyo"), + tzlocal_obj, +] +_sizes = [0, 1, 100, 10**4, 10**6] + + +class TimeIntsToPydatetime: + params = ( + ["time", "date", "datetime", "timestamp"], + _sizes, + _tzs, + ) + param_names = ["box", "size", "tz"] + # TODO: fold? + + def setup(self, box, size, tz): + if box == "date" and tz is not None: + # tz is ignored, so avoid running redundant benchmarks + raise NotImplementedError # skip benchmark + if size == 10**6 and tz is _tzs[-1]: + # This is cumbersomely-slow, so skip to trim runtime + raise NotImplementedError # skip benchmark + + arr = np.random.randint(0, 10, size=size, dtype="i8") + self.i8data = arr + + def time_ints_to_pydatetime(self, box, size, tz): + ints_to_pydatetime(self.i8data, tz, box=box) diff --git a/asv_bench/benchmarks/tslibs/tz_convert.py b/asv_bench/benchmarks/tslibs/tz_convert.py new file mode 100644 index 0000000000000..c87adb5e5d0e9 --- /dev/null +++ b/asv_bench/benchmarks/tslibs/tz_convert.py @@ -0,0 +1,53 @@ +from datetime import timezone + +import numpy as np + +from pandas._libs.tslibs.tzconversion import tz_localize_to_utc + +from .tslib import ( + _sizes, + _tzs, + tzlocal_obj, +) + +try: + old_sig = False + from pandas._libs.tslibs import tz_convert_from_utc +except ImportError: + try: + old_sig = False + from pandas._libs.tslibs.tzconversion import tz_convert_from_utc + except ImportError: + old_sig = True + from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc + + +class TimeTZConvert: + params = [ + _sizes, + [x for x in _tzs if x is not None], + ] + param_names = ["size", "tz"] + + def setup(self, size, tz): + if size == 10**6 and tz is tzlocal_obj: + # tzlocal is cumbersomely slow, so skip to keep runtime in check + raise NotImplementedError + + arr = np.random.randint(0, 10, size=size, dtype="i8") + self.i8data = arr + + def time_tz_convert_from_utc(self, size, tz): + # effectively: + # dti = DatetimeIndex(self.i8data, tz=tz) + # dti.tz_localize(None) + if old_sig: + tz_convert_from_utc(self.i8data, timezone.utc, tz) + else: + tz_convert_from_utc(self.i8data, tz) + + def time_tz_localize_to_utc(self, size, tz): + # effectively: + # dti = DatetimeIndex(self.i8data) + # dti.tz_localize(tz, ambiguous="NaT", nonexistent="NaT") + tz_localize_to_utc(self.i8data, tz, ambiguous="NaT", nonexistent="NaT") diff --git a/bench/alignment.py b/bench/alignment.py deleted file mode 100644 index bc3134f597ee0..0000000000000 --- a/bench/alignment.py +++ /dev/null @@ -1,22 +0,0 @@ -# Setup -from pandas.compat import range, lrange -import numpy as np -import pandas -import la -N = 1000 -K = 50 -arr1 = np.random.randn(N, K) -arr2 = np.random.randn(N, K) -idx1 = lrange(N) -idx2 = lrange(K) - -# pandas -dma1 = pandas.DataFrame(arr1, idx1, idx2) -dma2 = pandas.DataFrame(arr2, idx1[::-1], idx2[::-1]) - -# larry -lar1 = la.larry(arr1, [idx1, idx2]) -lar2 = la.larry(arr2, [idx1[::-1], idx2[::-1]]) - -for i in range(100): - result = lar1 + lar2 diff --git a/bench/bench_dense_to_sparse.py b/bench/bench_dense_to_sparse.py deleted file mode 100644 index e1dcd3456e88d..0000000000000 --- a/bench/bench_dense_to_sparse.py +++ /dev/null @@ -1,14 +0,0 @@ -from pandas import * - -K = 100 -N = 100000 -rng = DatetimeIndex('1/1/2000', periods=N, offset=datetools.Minute()) - -rng2 = np.asarray(rng).astype('M8[us]').astype('i8') - -series = {} -for i in range(1, K + 1): - data = np.random.randn(N)[:-i] - this_rng = rng2[:-i] - data[100:] = np.nan - series[i] = SparseSeries(data, index=this_rng) diff --git a/bench/bench_get_put_value.py b/bench/bench_get_put_value.py deleted file mode 100644 index 427e0b1b10a22..0000000000000 --- a/bench/bench_get_put_value.py +++ /dev/null @@ -1,56 +0,0 @@ -from pandas import * -from pandas.util.testing import rands -from pandas.compat import range - -N = 1000 -K = 50 - - -def _random_index(howmany): - return Index([rands(10) for _ in range(howmany)]) - -df = DataFrame(np.random.randn(N, K), index=_random_index(N), - columns=_random_index(K)) - - -def get1(): - for col in df.columns: - for row in df.index: - _ = df[col][row] - - -def get2(): - for col in df.columns: - for row in df.index: - _ = df.get_value(row, col) - - -def put1(): - for col in df.columns: - for row in df.index: - df[col][row] = 0 - - -def put2(): - for col in df.columns: - for row in df.index: - df.set_value(row, col, 0) - - -def resize1(): - buf = DataFrame() - for col in df.columns: - for row in df.index: - buf = buf.set_value(row, col, 5.) - return buf - - -def resize2(): - from collections import defaultdict - - buf = defaultdict(dict) - for col in df.columns: - for row in df.index: - buf[col][row] = 5. - - return DataFrame(buf) diff --git a/bench/bench_groupby.py b/bench/bench_groupby.py deleted file mode 100644 index d7a2853e1e7b2..0000000000000 --- a/bench/bench_groupby.py +++ /dev/null @@ -1,66 +0,0 @@ -from pandas import * -from pandas.util.testing import rands -from pandas.compat import range - -import string -import random - -k = 20000 -n = 10 - -foo = np.tile(np.array([rands(10) for _ in range(k)], dtype='O'), n) -foo2 = list(foo) -random.shuffle(foo) -random.shuffle(foo2) - -df = DataFrame({'A': foo, - 'B': foo2, - 'C': np.random.randn(n * k)}) - -import pandas._sandbox as sbx - - -def f(): - table = sbx.StringHashTable(len(df)) - ret = table.factorize(df['A']) - return ret - - -def g(): - table = sbx.PyObjectHashTable(len(df)) - ret = table.factorize(df['A']) - return ret - -ret = f() - -""" -import pandas._tseries as lib - -f = np.std - - -grouped = df.groupby(['A', 'B']) - -label_list = [ping.labels for ping in grouped.groupings] -shape = [len(ping.ids) for ping in grouped.groupings] - -from pandas.core.groupby import get_group_index - - -group_index = get_group_index(label_list, shape, - sort=True, xnull=True).astype('i4') - -ngroups = np.prod(shape) - -indexer = lib.groupsort_indexer(group_index, ngroups) - -values = df['C'].values.take(indexer) -group_index = group_index.take(indexer) - -f = lambda x: x.std(ddof=1) - -grouper = lib.Grouper(df['C'], np.ndarray.std, group_index, ngroups) -result = grouper.get_result() - -expected = grouped.std() -""" diff --git a/bench/bench_join_panel.py b/bench/bench_join_panel.py deleted file mode 100644 index f3c3f8ba15f70..0000000000000 --- a/bench/bench_join_panel.py +++ /dev/null @@ -1,85 +0,0 @@ -# reasonably efficient - - -def create_panels_append(cls, panels): - """ return an append list of panels """ - panels = [a for a in panels if a is not None] - # corner cases - if len(panels) == 0: - return None - elif len(panels) == 1: - return panels[0] - elif len(panels) == 2 and panels[0] == panels[1]: - return panels[0] - # import pdb; pdb.set_trace() - # create a joint index for the axis - - def joint_index_for_axis(panels, axis): - s = set() - for p in panels: - s.update(list(getattr(p, axis))) - return sorted(list(s)) - - def reindex_on_axis(panels, axis, axis_reindex): - new_axis = joint_index_for_axis(panels, axis) - new_panels = [p.reindex(**{axis_reindex: new_axis, - 'copy': False}) for p in panels] - return new_panels, new_axis - # create the joint major index, dont' reindex the sub-panels - we are - # appending - major = joint_index_for_axis(panels, 'major_axis') - # reindex on minor axis - panels, minor = reindex_on_axis(panels, 'minor_axis', 'minor') - # reindex on items - panels, items = reindex_on_axis(panels, 'items', 'items') - # concatenate values - try: - values = np.concatenate([p.values for p in panels], axis=1) - except Exception as detail: - raise Exception("cannot append values that dont' match dimensions! -> [%s] %s" - % (','.join(["%s" % p for p in panels]), str(detail))) - # pm('append - create_panel') - p = Panel(values, items=items, major_axis=major, - minor_axis=minor) - # pm('append - done') - return p - - -# does the job but inefficient (better to handle like you read a table in -# pytables...e.g create a LongPanel then convert to Wide) -def create_panels_join(cls, panels): - """ given an array of panels's, create a single panel """ - panels = [a for a in panels if a is not None] - # corner cases - if len(panels) == 0: - return None - elif len(panels) == 1: - return panels[0] - elif len(panels) == 2 and panels[0] == panels[1]: - return panels[0] - d = dict() - minor, major, items = set(), set(), set() - for panel in panels: - items.update(panel.items) - major.update(panel.major_axis) - minor.update(panel.minor_axis) - values = panel.values - for item, item_index in panel.items.indexMap.items(): - for minor_i, minor_index in panel.minor_axis.indexMap.items(): - for major_i, major_index in panel.major_axis.indexMap.items(): - try: - d[(minor_i, major_i, item)] = values[item_index, major_index, minor_index] - except: - pass - # stack the values - minor = sorted(list(minor)) - major = sorted(list(major)) - items = sorted(list(items)) - # create the 3d stack (items x columns x indicies) - data = np.dstack([np.asarray([np.asarray([d.get((minor_i, major_i, item), np.nan) - for item in items]) - for major_i in major]).transpose() - for minor_i in minor]) - # construct the panel - return Panel(data, items, major, minor) -add_class_method(Panel, create_panels_join, 'join_many') diff --git a/bench/bench_khash_dict.py b/bench/bench_khash_dict.py deleted file mode 100644 index 054fc36131b65..0000000000000 --- a/bench/bench_khash_dict.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -Some comparisons of khash.h to Python dict -""" -from __future__ import print_function - -import numpy as np -import os - -from vbench.api import Benchmark -from pandas.util.testing import rands -from pandas.compat import range -import pandas._tseries as lib -import pandas._sandbox as sbx -import time - -import psutil - -pid = os.getpid() -proc = psutil.Process(pid) - - -def object_test_data(n): - pass - - -def string_test_data(n): - return np.array([rands(10) for _ in range(n)], dtype='O') - - -def int_test_data(n): - return np.arange(n, dtype='i8') - -N = 1000000 - -#---------------------------------------------------------------------- -# Benchmark 1: map_locations - - -def map_locations_python_object(): - arr = string_test_data(N) - return _timeit(lambda: lib.map_indices_object(arr)) - - -def map_locations_khash_object(): - arr = string_test_data(N) - - def f(): - table = sbx.PyObjectHashTable(len(arr)) - table.map_locations(arr) - return _timeit(f) - - -def _timeit(f, iterations=10): - start = time.time() - for _ in range(iterations): - foo = f() - elapsed = time.time() - start - return elapsed - -#---------------------------------------------------------------------- -# Benchmark 2: lookup_locations - - -def lookup_python(values): - table = lib.map_indices_object(values) - return _timeit(lambda: lib.merge_indexer_object(values, table)) - - -def lookup_khash(values): - table = sbx.PyObjectHashTable(len(values)) - table.map_locations(values) - locs = table.lookup_locations(values) - # elapsed = _timeit(lambda: table.lookup_locations2(values)) - return table - - -def leak(values): - for _ in range(100): - print(proc.get_memory_info()) - table = lookup_khash(values) - # table.destroy() - -arr = string_test_data(N) - -#---------------------------------------------------------------------- -# Benchmark 3: unique - -#---------------------------------------------------------------------- -# Benchmark 4: factorize diff --git a/bench/bench_merge.R b/bench/bench_merge.R deleted file mode 100644 index 3ed4618494857..0000000000000 --- a/bench/bench_merge.R +++ /dev/null @@ -1,161 +0,0 @@ -library(plyr) -library(data.table) -N <- 10000 -indices = rep(NA, N) -indices2 = rep(NA, N) -for (i in 1:N) { - indices[i] <- paste(sample(letters, 10), collapse="") - indices2[i] <- paste(sample(letters, 10), collapse="") -} -left <- data.frame(key=rep(indices[1:8000], 10), - key2=rep(indices2[1:8000], 10), - value=rnorm(80000)) -right <- data.frame(key=indices[2001:10000], - key2=indices2[2001:10000], - value2=rnorm(8000)) - -right2 <- data.frame(key=rep(right$key, 2), - key2=rep(right$key2, 2), - value2=rnorm(16000)) - -left.dt <- data.table(left, key=c("key", "key2")) -right.dt <- data.table(right, key=c("key", "key2")) -right2.dt <- data.table(right2, key=c("key", "key2")) - -# left.dt2 <- data.table(left) -# right.dt2 <- data.table(right) - -## left <- data.frame(key=rep(indices[1:1000], 10), -## key2=rep(indices2[1:1000], 10), -## value=rnorm(100000)) -## right <- data.frame(key=indices[1:1000], -## key2=indices2[1:1000], -## value2=rnorm(10000)) - -timeit <- function(func, niter=10) { - timing = rep(NA, niter) - for (i in 1:niter) { - gc() - timing[i] <- system.time(func())[3] - } - mean(timing) -} - -left.join <- function(sort=FALSE) { - result <- base::merge(left, right, all.x=TRUE, sort=sort) -} - -right.join <- function(sort=FALSE) { - result <- base::merge(left, right, all.y=TRUE, sort=sort) -} - -outer.join <- function(sort=FALSE) { - result <- base::merge(left, right, all=TRUE, sort=sort) -} - -inner.join <- function(sort=FALSE) { - result <- base::merge(left, right, all=FALSE, sort=sort) -} - -left.join.dt <- function(sort=FALSE) { - result <- right.dt[left.dt] -} - -right.join.dt <- function(sort=FALSE) { - result <- left.dt[right.dt] -} - -outer.join.dt <- function(sort=FALSE) { - result <- merge(left.dt, right.dt, all=TRUE, sort=sort) -} - -inner.join.dt <- function(sort=FALSE) { - result <- merge(left.dt, right.dt, all=FALSE, sort=sort) -} - -plyr.join <- function(type) { - result <- plyr::join(left, right, by=c("key", "key2"), - type=type, match="first") -} - -sort.options <- c(FALSE, TRUE) - -# many-to-one - -results <- matrix(nrow=4, ncol=3) -colnames(results) <- c("base::merge", "plyr", "data.table") -rownames(results) <- c("inner", "outer", "left", "right") - -base.functions <- c(inner.join, outer.join, left.join, right.join) -plyr.functions <- c(function() plyr.join("inner"), - function() plyr.join("full"), - function() plyr.join("left"), - function() plyr.join("right")) -dt.functions <- c(inner.join.dt, outer.join.dt, left.join.dt, right.join.dt) -for (i in 1:4) { - base.func <- base.functions[[i]] - plyr.func <- plyr.functions[[i]] - dt.func <- dt.functions[[i]] - results[i, 1] <- timeit(base.func) - results[i, 2] <- timeit(plyr.func) - results[i, 3] <- timeit(dt.func) -} - - -# many-to-many - -left.join <- function(sort=FALSE) { - result <- base::merge(left, right2, all.x=TRUE, sort=sort) -} - -right.join <- function(sort=FALSE) { - result <- base::merge(left, right2, all.y=TRUE, sort=sort) -} - -outer.join <- function(sort=FALSE) { - result <- base::merge(left, right2, all=TRUE, sort=sort) -} - -inner.join <- function(sort=FALSE) { - result <- base::merge(left, right2, all=FALSE, sort=sort) -} - -left.join.dt <- function(sort=FALSE) { - result <- right2.dt[left.dt] -} - -right.join.dt <- function(sort=FALSE) { - result <- left.dt[right2.dt] -} - -outer.join.dt <- function(sort=FALSE) { - result <- merge(left.dt, right2.dt, all=TRUE, sort=sort) -} - -inner.join.dt <- function(sort=FALSE) { - result <- merge(left.dt, right2.dt, all=FALSE, sort=sort) -} - -sort.options <- c(FALSE, TRUE) - -# many-to-one - -results <- matrix(nrow=4, ncol=3) -colnames(results) <- c("base::merge", "plyr", "data.table") -rownames(results) <- c("inner", "outer", "left", "right") - -base.functions <- c(inner.join, outer.join, left.join, right.join) -plyr.functions <- c(function() plyr.join("inner"), - function() plyr.join("full"), - function() plyr.join("left"), - function() plyr.join("right")) -dt.functions <- c(inner.join.dt, outer.join.dt, left.join.dt, right.join.dt) -for (i in 1:4) { - base.func <- base.functions[[i]] - plyr.func <- plyr.functions[[i]] - dt.func <- dt.functions[[i]] - results[i, 1] <- timeit(base.func) - results[i, 2] <- timeit(plyr.func) - results[i, 3] <- timeit(dt.func) -} - diff --git a/bench/bench_merge.py b/bench/bench_merge.py deleted file mode 100644 index 330dba7b9af69..0000000000000 --- a/bench/bench_merge.py +++ /dev/null @@ -1,105 +0,0 @@ -import random -import gc -import time -from pandas import * -from pandas.compat import range, lrange, StringIO -from pandas.util.testing import rands - -N = 10000 -ngroups = 10 - - -def get_test_data(ngroups=100, n=N): - unique_groups = lrange(ngroups) - arr = np.asarray(np.tile(unique_groups, n / ngroups), dtype=object) - - if len(arr) < n: - arr = np.asarray(list(arr) + unique_groups[:n - len(arr)], - dtype=object) - - random.shuffle(arr) - return arr - -# aggregate multiple columns -# df = DataFrame({'key1' : get_test_data(ngroups=ngroups), -# 'key2' : get_test_data(ngroups=ngroups), -# 'data1' : np.random.randn(N), -# 'data2' : np.random.randn(N)}) - -# df2 = DataFrame({'key1' : get_test_data(ngroups=ngroups, n=N//10), -# 'key2' : get_test_data(ngroups=ngroups//2, n=N//10), -# 'value' : np.random.randn(N // 10)}) -# result = merge.merge(df, df2, on='key2') - -N = 10000 - -indices = np.array([rands(10) for _ in range(N)], dtype='O') -indices2 = np.array([rands(10) for _ in range(N)], dtype='O') -key = np.tile(indices[:8000], 10) -key2 = np.tile(indices2[:8000], 10) - -left = DataFrame({'key': key, 'key2': key2, - 'value': np.random.randn(80000)}) -right = DataFrame({'key': indices[2000:], 'key2': indices2[2000:], - 'value2': np.random.randn(8000)}) - -right2 = right.append(right, ignore_index=True) - - -join_methods = ['inner', 'outer', 'left', 'right'] -results = DataFrame(index=join_methods, columns=[False, True]) -niter = 10 -for sort in [False, True]: - for join_method in join_methods: - f = lambda: merge(left, right, how=join_method, sort=sort) - gc.disable() - start = time.time() - for _ in range(niter): - f() - elapsed = (time.time() - start) / niter - gc.enable() - results[sort][join_method] = elapsed -# results.columns = ['pandas'] -results.columns = ['dont_sort', 'sort'] - - -# R results -# many to one -r_results = read_table(StringIO(""" base::merge plyr data.table -inner 0.2475 0.1183 0.1100 -outer 0.4213 0.1916 0.2090 -left 0.2998 0.1188 0.0572 -right 0.3102 0.0536 0.0376 -"""), sep='\s+') - -presults = results[['dont_sort']].rename(columns={'dont_sort': 'pandas'}) -all_results = presults.join(r_results) - -all_results = all_results.div(all_results['pandas'], axis=0) - -all_results = all_results.ix[:, ['pandas', 'data.table', 'plyr', - 'base::merge']] - -sort_results = DataFrame.from_items([('pandas', results['sort']), - ('R', r_results['base::merge'])]) -sort_results['Ratio'] = sort_results['R'] / sort_results['pandas'] - - -nosort_results = DataFrame.from_items([('pandas', results['dont_sort']), - ('R', r_results['base::merge'])]) -nosort_results['Ratio'] = nosort_results['R'] / nosort_results['pandas'] - -# many to many - -# many to one -r_results = read_table(StringIO("""base::merge plyr data.table -inner 0.4610 0.1276 0.1269 -outer 0.9195 0.1881 0.2725 -left 0.6559 0.1257 0.0678 -right 0.6425 0.0522 0.0428 -"""), sep='\s+') - -all_results = presults.join(r_results) -all_results = all_results.div(all_results['pandas'], axis=0) -all_results = all_results.ix[:, ['pandas', 'data.table', 'plyr', - 'base::merge']] diff --git a/bench/bench_merge_sqlite.py b/bench/bench_merge_sqlite.py deleted file mode 100644 index 3ad4b810119c3..0000000000000 --- a/bench/bench_merge_sqlite.py +++ /dev/null @@ -1,87 +0,0 @@ -import numpy as np -from collections import defaultdict -import gc -import time -from pandas import DataFrame -from pandas.util.testing import rands -from pandas.compat import range, zip -import random - -N = 10000 - -indices = np.array([rands(10) for _ in range(N)], dtype='O') -indices2 = np.array([rands(10) for _ in range(N)], dtype='O') -key = np.tile(indices[:8000], 10) -key2 = np.tile(indices2[:8000], 10) - -left = DataFrame({'key': key, 'key2': key2, - 'value': np.random.randn(80000)}) -right = DataFrame({'key': indices[2000:], 'key2': indices2[2000:], - 'value2': np.random.randn(8000)}) - -# right2 = right.append(right, ignore_index=True) -# right = right2 - -# random.shuffle(key2) -# indices2 = indices.copy() -# random.shuffle(indices2) - -# Prepare Database -import sqlite3 -create_sql_indexes = True - -conn = sqlite3.connect(':memory:') -conn.execute( - 'create table left( key varchar(10), key2 varchar(10), value int);') -conn.execute( - 'create table right( key varchar(10), key2 varchar(10), value2 int);') -conn.executemany('insert into left values (?, ?, ?)', - zip(key, key2, left['value'])) -conn.executemany('insert into right values (?, ?, ?)', - zip(right['key'], right['key2'], right['value2'])) - -# Create Indices -if create_sql_indexes: - conn.execute('create index left_ix on left(key, key2)') - conn.execute('create index right_ix on right(key, key2)') - - -join_methods = ['inner', 'left outer', 'left'] # others not supported -sql_results = DataFrame(index=join_methods, columns=[False]) -niter = 5 -for sort in [False]: - for join_method in join_methods: - sql = """CREATE TABLE test as select * - from left - %s join right - on left.key=right.key - and left.key2 = right.key2;""" % join_method - sql = """select * - from left - %s join right - on left.key=right.key - and left.key2 = right.key2;""" % join_method - - if sort: - sql = '%s order by key, key2' % sql - f = lambda: list(conn.execute(sql)) # list fetches results - g = lambda: conn.execute(sql) # list fetches results - gc.disable() - start = time.time() - # for _ in range(niter): - g() - elapsed = (time.time() - start) / niter - gc.enable() - - cur = conn.execute("DROP TABLE test") - conn.commit() - - sql_results[sort][join_method] = elapsed - sql_results.columns = ['sqlite3'] # ['dont_sort', 'sort'] - sql_results.index = ['inner', 'outer', 'left'] - - sql = """select * - from left - inner join right - on left.key=right.key - and left.key2 = right.key2;""" diff --git a/bench/bench_pivot.R b/bench/bench_pivot.R deleted file mode 100644 index 06dc6a105bc43..0000000000000 --- a/bench/bench_pivot.R +++ /dev/null @@ -1,27 +0,0 @@ -library(reshape2) - - -n <- 100000 -a.size <- 5 -b.size <- 5 - -data <- data.frame(a=sample(letters[1:a.size], n, replace=T), - b=sample(letters[1:b.size], n, replace=T), - c=rnorm(n), - d=rnorm(n)) - -timings <- numeric() - -# acast(melt(data, id=c("a", "b")), a ~ b, mean) -# acast(melt(data, id=c("a", "b")), a + b ~ variable, mean) - -for (i in 1:10) { - gc() - tim <- system.time(acast(melt(data, id=c("a", "b")), a ~ b, mean, - subset=.(variable=="c"))) - timings[i] = tim[3] -} - -mean(timings) - -acast(melt(data, id=c("a", "b")), a ~ b, mean, subset=.(variable="c")) diff --git a/bench/bench_pivot.py b/bench/bench_pivot.py deleted file mode 100644 index 007bd0aaebc2f..0000000000000 --- a/bench/bench_pivot.py +++ /dev/null @@ -1,16 +0,0 @@ -from pandas import * -import string - - -n = 100000 -asize = 5 -bsize = 5 - -letters = np.asarray(list(string.letters), dtype=object) - -data = DataFrame(dict(foo=letters[:asize][np.random.randint(0, asize, n)], - bar=letters[:bsize][np.random.randint(0, bsize, n)], - baz=np.random.randn(n), - qux=np.random.randn(n))) - -table = pivot_table(data, xby=['foo', 'bar']) diff --git a/bench/bench_sparse.py b/bench/bench_sparse.py deleted file mode 100644 index 7dc2db05cfe20..0000000000000 --- a/bench/bench_sparse.py +++ /dev/null @@ -1,93 +0,0 @@ -import sys -import numpy as np - -from pandas import * -import pandas.core.sparse as spm -import pandas.compat as compat -reload(spm) -from pandas.core.sparse import * - -N = 10000. - -arr1 = np.arange(N) -index = Index(np.arange(N)) - -off = N // 10 -arr1[off: 2 * off] = np.NaN -arr1[4 * off: 5 * off] = np.NaN -arr1[8 * off: 9 * off] = np.NaN - -arr2 = np.arange(N) -arr2[3 * off // 2: 2 * off + off // 2] = np.NaN -arr2[8 * off + off // 2: 9 * off + off // 2] = np.NaN - -s1 = SparseSeries(arr1, index=index) -s2 = SparseSeries(arr2, index=index) - -is1 = SparseSeries(arr1, kind='integer', index=index) -is2 = SparseSeries(arr2, kind='integer', index=index) - -s1_dense = s1.to_dense() -s2_dense = s2.to_dense() - -if 'linux' in sys.platform: - pth = '/home/wesm/code/pandas/example' -else: - pth = '/Users/wesm/code/pandas/example' - -dm = DataFrame.load(pth) - -sdf = dm.to_sparse() - - -def new_data_like(sdf): - new_data = {} - for col, series in compat.iteritems(sdf): - new_data[col] = SparseSeries(np.random.randn(len(series.sp_values)), - index=sdf.index, - sparse_index=series.sp_index, - fill_value=series.fill_value) - - return SparseDataFrame(new_data) - -# data = {} -# for col, ser in dm.iteritems(): -# data[col] = SparseSeries(ser) - -dwp = Panel.fromDict({'foo': dm}) -# sdf = SparseDataFrame(data) - - -lp = stack_sparse_frame(sdf) - - -swp = SparsePanel({'A': sdf}) -swp = SparsePanel({'A': sdf, - 'B': sdf, - 'C': sdf, - 'D': sdf}) - -y = sdf -x = SparsePanel({'x1': sdf + new_data_like(sdf) / 10, - 'x2': sdf + new_data_like(sdf) / 10}) - -dense_y = sdf -dense_x = x.to_dense() - -# import hotshot, hotshot.stats -# prof = hotshot.Profile('test.prof') - -# benchtime, stones = prof.runcall(ols, y=y, x=x) - -# prof.close() - -# stats = hotshot.stats.load('test.prof') - -dense_model = ols(y=dense_y, x=dense_x) - -import pandas.stats.plm as plm -import pandas.stats.interface as face -reload(plm) -reload(face) - -# model = face.ols(y=y, x=x) diff --git a/bench/bench_take_indexing.py b/bench/bench_take_indexing.py deleted file mode 100644 index 5fb584bcfe45f..0000000000000 --- a/bench/bench_take_indexing.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import print_function -import numpy as np - -from pandas import * -import pandas._tseries as lib - -from pandas import DataFrame -import timeit -from pandas.compat import zip - -setup = """ -from pandas import Series -import pandas._tseries as lib -import random -import numpy as np - -import random -n = %d -k = %d -arr = np.random.randn(n, k) -indexer = np.arange(n, dtype=np.int32) -indexer = indexer[::-1] -""" - -sizes = [100, 1000, 10000, 100000] -iters = [1000, 1000, 100, 1] - -fancy_2d = [] -take_2d = [] -cython_2d = [] - -n = 1000 - - -def _timeit(stmt, size, k=5, iters=1000): - timer = timeit.Timer(stmt=stmt, setup=setup % (sz, k)) - return timer.timeit(n) / n - -for sz, its in zip(sizes, iters): - print(sz) - fancy_2d.append(_timeit('arr[indexer]', sz, iters=its)) - take_2d.append(_timeit('arr.take(indexer, axis=0)', sz, iters=its)) - cython_2d.append(_timeit('lib.take_axis0(arr, indexer)', sz, iters=its)) - -df = DataFrame({'fancy': fancy_2d, - 'take': take_2d, - 'cython': cython_2d}) - -print(df) - -from pandas.rpy.common import r -r('mat <- matrix(rnorm(50000), nrow=10000, ncol=5)') -r('set.seed(12345') -r('indexer <- sample(1:10000)') -r('mat[indexer,]') diff --git a/bench/bench_unique.py b/bench/bench_unique.py deleted file mode 100644 index 87bd2f2df586c..0000000000000 --- a/bench/bench_unique.py +++ /dev/null @@ -1,278 +0,0 @@ -from __future__ import print_function -from pandas import * -from pandas.util.testing import rands -from pandas.compat import range, zip -import pandas._tseries as lib -import numpy as np -import matplotlib.pyplot as plt - -N = 50000 -K = 10000 - -groups = np.array([rands(10) for _ in range(K)], dtype='O') -groups2 = np.array([rands(10) for _ in range(K)], dtype='O') - -labels = np.tile(groups, N // K) -labels2 = np.tile(groups2, N // K) -data = np.random.randn(N) - - -def timeit(f, niter): - import gc - import time - gc.disable() - start = time.time() - for _ in range(niter): - f() - elapsed = (time.time() - start) / niter - gc.enable() - return elapsed - - -def algo1(): - unique_labels = np.unique(labels) - result = np.empty(len(unique_labels)) - for i, label in enumerate(unique_labels): - result[i] = data[labels == label].sum() - - -def algo2(): - unique_labels = np.unique(labels) - indices = lib.groupby_indices(labels) - result = np.empty(len(unique_labels)) - - for i, label in enumerate(unique_labels): - result[i] = data.take(indices[label]).sum() - - -def algo3_nosort(): - rizer = lib.DictFactorizer() - labs, counts = rizer.factorize(labels, sort=False) - k = len(rizer.uniques) - out = np.empty(k) - lib.group_add(out, counts, data, labs) - - -def algo3_sort(): - rizer = lib.DictFactorizer() - labs, counts = rizer.factorize(labels, sort=True) - k = len(rizer.uniques) - out = np.empty(k) - lib.group_add(out, counts, data, labs) - -import numpy as np -import random - - -# dict to hold results -counts = {} - -# a hack to generate random key, value pairs. -# 5k keys, 100k values -x = np.tile(np.arange(5000, dtype='O'), 20) -random.shuffle(x) -xarr = x -x = [int(y) for y in x] -data = np.random.uniform(0, 1, 100000) - - -def f(): - # groupby sum - for k, v in zip(x, data): - try: - counts[k] += v - except KeyError: - counts[k] = v - - -def f2(): - rizer = lib.DictFactorizer() - labs, counts = rizer.factorize(xarr, sort=False) - k = len(rizer.uniques) - out = np.empty(k) - lib.group_add(out, counts, data, labs) - - -def algo4(): - rizer = lib.DictFactorizer() - labs1, _ = rizer.factorize(labels, sort=False) - k1 = len(rizer.uniques) - - rizer = lib.DictFactorizer() - labs2, _ = rizer.factorize(labels2, sort=False) - k2 = len(rizer.uniques) - - group_id = labs1 * k2 + labs2 - max_group = k1 * k2 - - if max_group > 1e6: - rizer = lib.Int64Factorizer(len(group_id)) - group_id, _ = rizer.factorize(group_id.astype('i8'), sort=True) - max_group = len(rizer.uniques) - - out = np.empty(max_group) - counts = np.zeros(max_group, dtype='i4') - lib.group_add(out, counts, data, group_id) - -# cumtime percall filename:lineno(function) -# 0.592 0.592
This API page has moved here.
+This API page has moved here.
- \ No newline at end of file +