{"id":1113200,"date":"2025-01-08T17:45:34","date_gmt":"2025-01-08T09:45:34","guid":{"rendered":"https:\/\/docs.pingcode.com\/ask\/ask-ask\/1113200.html"},"modified":"2025-01-08T17:45:37","modified_gmt":"2025-01-08T09:45:37","slug":"python%e6%95%b0%e6%8d%ae%e5%88%86%e6%9e%90%e5%a6%82%e4%bd%95%e5%ae%9e%e7%8e%b0%e5%88%86%e5%b8%83%e5%bc%8f","status":"publish","type":"post","link":"https:\/\/docs.pingcode.com\/ask\/1113200.html","title":{"rendered":"python\u6570\u636e\u5206\u6790\u5982\u4f55\u5b9e\u73b0\u5206\u5e03\u5f0f"},"content":{"rendered":"

\"python\u6570\u636e\u5206\u6790\u5982\u4f55\u5b9e\u73b0\u5206\u5e03\u5f0f\"<\/p>\n

Python\u6570\u636e\u5206\u6790\u5b9e\u73b0\u5206\u5e03\u5f0f\u65b9\u6cd5\u5305\u62ec\uff1a\u4f7f\u7528Dask\u3001\u5229\u7528Spark\u3001\u4f7f\u7528Ray\u3001\u7ed3\u5408Celery\u7b49\u3002\u672c\u6587\u5c06\u8be6\u7ec6\u4ecb\u7ecd\u4f7f\u7528Dask\u5b9e\u73b0\u5206\u5e03\u5f0f\u6570\u636e\u5206\u6790\u7684\u65b9\u6cd5\u3002<\/strong><\/p>\n<\/p>\n

\u4e00\u3001DASK\u6982\u8ff0<\/p>\n

Dask\u662f\u4e00\u4e2a\u7075\u6d3b\u7684\u5e76\u884c\u8ba1\u7b97\u5e93\uff0c\u9002\u7528\u4e8ePython\u3002\u5b83\u901a\u8fc7\u5c06\u4efb\u52a1\u5206\u89e3\u6210\u66f4\u5c0f\u7684\u4efb\u52a1\u5e76\u5206\u914d\u5230\u591a\u4e2a\u8ba1\u7b97\u673a\u6838\u5fc3\u4e0a\u6765\u6267\u884c\uff0c\u4ece\u800c\u5b9e\u73b0\u9ad8\u6548\u7684\u5206\u5e03\u5f0f\u8ba1\u7b97\u3002Dask\u652f\u6301\u5927\u591a\u6570Pandas\u64cd\u4f5c\uff0c\u56e0\u6b64\u5bf9\u4e8e\u719f\u6089Pandas\u7684\u7528\u6237\u6765\u8bf4\uff0c\u4f7f\u7528Dask\u975e\u5e38\u65b9\u4fbf\u3002<\/p>\n<\/p>\n

Dask\u5177\u5907\u4ee5\u4e0b\u51e0\u4e2a\u7279\u70b9\uff1a<\/strong><\/p>\n<\/p>\n

    \n
  1. \u7075\u6d3b\u6027<\/strong>\uff1a\u652f\u6301\u81ea\u5b9a\u4e49\u7684\u5e76\u884c\u8ba1\u7b97\u4efb\u52a1\u3002<\/li>\n
  2. \u517c\u5bb9\u6027<\/strong>\uff1a\u4e0ePandas\u548cNumPy\u6570\u636e\u7ed3\u6784\u517c\u5bb9\u3002<\/li>\n
  3. \u6269\u5c55\u6027<\/strong>\uff1a\u652f\u6301\u4ece\u5355\u673a\u5230\u96c6\u7fa4\u7684\u6269\u5c55\u3002<\/li>\n
  4. \u8c03\u5ea6\u7075\u6d3b<\/strong>\uff1a\u652f\u6301\u591a\u79cd\u8c03\u5ea6\u5668\uff0c\u53ef\u4ee5\u6839\u636e\u9700\u6c42\u9009\u62e9\u5408\u9002\u7684\u8c03\u5ea6\u5668\u3002<\/li>\n<\/ol>\n

    \u4e8c\u3001DASK\u7684\u5b89\u88c5\u4e0e\u57fa\u672c\u4f7f\u7528<\/p>\n<\/p>\n

      \n
    1. \u5b89\u88c5Dask<\/strong><\/li>\n<\/ol>\n

      pip install dask[complete]<\/p>\n
      <\/code><\/pre>\n<\/p>\n
        \n
      1. \u521b\u5efaDask DataFrame<\/strong><\/p>\n
        Dask DataFrame\u7c7b\u4f3c\u4e8ePandas DataFrame\uff0c\u4f46\u5b83\u662f\u6309\u5757\u5206\u5272\u7684\uff0c\u56e0\u6b64\u53ef\u4ee5\u5e76\u884c\u5904\u7406\u3002<\/li>\n<\/p>\n<\/ol>\n
        import dask.dataframe as dd<\/p>\n
        \u8bfb\u53d6CSV\u6587\u4ef6\u521b\u5efaDask DataFrame<\/strong><\/h2>\n
        df = dd.read_csv('large_dataset.csv')<\/p>\n
        <\/code><\/pre>\n<\/p>\n
          \n
        1. \u6267\u884c\u57fa\u672c\u64cd\u4f5c<\/strong><\/p>\n
          Dask DataFrame\u652f\u6301\u5927\u591a\u6570Pandas\u64cd\u4f5c\uff0c\u5982\u8fc7\u6ee4\u3001\u805a\u5408\u3001\u5206\u7ec4\u7b49\u3002<\/li>\n<\/p>\n<\/ol>\n
          # \u57fa\u672c\u64cd\u4f5c<\/p>\n
          filtered_df = df[df['column'] > 100]<\/p>\n
          grouped_df = filtered_df.groupby('category').sum()<\/p>\n
          \u6267\u884c\u8ba1\u7b97\u5e76\u83b7\u53d6\u7ed3\u679c<\/strong><\/h2>\n
          result = grouped_df.compute()<\/p>\n
          print(result)<\/p>\n
          <\/code><\/pre>\n<\/p>\n
          Dask\u7684\u8ba1\u7b97\u662f\u60f0\u6027\u7684\uff0c\u53ea\u6709\u5728\u8c03\u7528compute()<\/code>\u65b9\u6cd5\u65f6\u624d\u4f1a\u771f\u6b63\u6267\u884c\u8ba1\u7b97\u3002<\/p>\n<\/p>\n
          \u4e09\u3001DASK\u4efb\u52a1\u8c03\u5ea6<\/p>\n
          Dask\u652f\u6301\u591a\u79cd\u8c03\u5ea6\u5668\uff0c\u5305\u62ec\u5355\u673a\u8c03\u5ea6\u5668\u3001\u5206\u5e03\u5f0f\u8c03\u5ea6\u5668\u7b49\u3002\u6839\u636e\u9700\u6c42\u9009\u62e9\u5408\u9002\u7684\u8c03\u5ea6\u5668\uff0c\u53ef\u4ee5\u6700\u5927\u5316\u5229\u7528\u8ba1\u7b97\u8d44\u6e90\u3002<\/p>\n<\/p>\n
            \n
          1. \u5355\u673a\u8c03\u5ea6\u5668<\/strong><\/p>\n
            \u9002\u7528\u4e8e\u5355\u673a\u591a\u6838\u73af\u5883\uff0c\u4f7f\u7528\u7b80\u5355\uff0c\u9002\u5408\u5f00\u53d1\u548c\u8c03\u8bd5\u3002<\/li>\n<\/p>\n<\/ol>\n
            import dask<\/p>\n
            \u4f7f\u7528\u5355\u673a\u8c03\u5ea6\u5668<\/strong><\/h2>\n
            dask.config.set(scheduler='threads')<\/p>\n
            <\/code><\/pre>\n<\/p>\n
              \n
            1. \u5206\u5e03\u5f0f\u8c03\u5ea6\u5668<\/strong><\/p>\n
              \u9002\u7528\u4e8e\u96c6\u7fa4\u73af\u5883\uff0c\u901a\u8fc7Dask.distributed\u5e93\u5b9e\u73b0\u3002<\/li>\n<\/p>\n<\/ol>\n
              from dask.distributed import Client<\/p>\n
              \u521b\u5efaDask\u5206\u5e03\u5f0f\u5ba2\u6237\u7aef<\/strong><\/h2>\n
              client = Client('scheduler_address:8786')<\/p>\n
              \u73b0\u5728\u53ef\u4ee5\u4f7f\u7528\u5206\u5e03\u5f0f\u8ba1\u7b97\u8d44\u6e90\u6267\u884c\u4efb\u52a1<\/strong><\/h2>\n
              result = df.compute()<\/p>\n
              print(result)<\/p>\n
              <\/code><\/pre>\n<\/p>\n
              \u56db\u3001DASK\u9ad8\u7ea7\u529f\u80fd<\/p>\n<\/p>\n
                \n
              1. Delayed\u5bf9\u8c61<\/strong><\/p>\n
                Dask Delayed\u5bf9\u8c61\u5141\u8bb8\u5c06\u81ea\u5b9a\u4e49\u51fd\u6570\u8f6c\u5316\u4e3a\u5e76\u884c\u8ba1\u7b97\u4efb\u52a1\u3002<\/li>\n<\/p>\n<\/ol>\n
                from dask import delayed<\/p>\n
                \u81ea\u5b9a\u4e49\u51fd\u6570<\/strong><\/h2>\n
                def inc(x):<\/p>\n
                    return x + 1<\/p>\n
                \u4f7f\u7528Delayed\u5bf9\u8c61\u521b\u5efa\u4efb\u52a1<\/strong><\/h2>\n
                delayed_inc = delayed(inc)<\/p>\n
                \u6267\u884c\u5e76\u884c\u8ba1\u7b97<\/strong><\/h2>\n
                result = delayed_inc(10).compute()<\/p>\n
                print(result)<\/p>\n
                <\/code><\/pre>\n<\/p>\n
                  \n
                1. Bag\u6570\u636e\u7ed3\u6784<\/strong><\/p>\n
                  Dask Bag\u7c7b\u4f3c\u4e8ePython\u7684\u5217\u8868\uff0c\u9002\u7528\u4e8e\u5904\u7406\u975e\u7ed3\u6784\u5316\u6216\u534a\u7ed3\u6784\u5316\u6570\u636e\u3002<\/li>\n<\/p>\n<\/ol>\n
                  import dask.bag as db<\/p>\n
                  \u521b\u5efaDask Bag<\/strong><\/h2>\n
                  bag = db.from_sequence([1, 2, 3, 4, 5])<\/p>\n
                  \u6267\u884c\u5e76\u884c\u8ba1\u7b97<\/strong><\/h2>\n
                  result = bag.map(lambda x: x * 2).compute()<\/p>\n
                  print(result)<\/p>\n
                  <\/code><\/pre>\n<\/p>\n
                    \n
                  1. Array\u6570\u636e\u7ed3\u6784<\/strong><\/p>\n
                    Dask Array\u7c7b\u4f3c\u4e8eNumPy\u6570\u7ec4\uff0c\u9002\u7528\u4e8e\u5927\u89c4\u6a21\u6570\u503c\u8ba1\u7b97\u3002<\/li>\n<\/p>\n<\/ol>\n
                    import dask.array as da<\/p>\n
                    \u521b\u5efaDask Array<\/strong><\/h2>\n
                    array = da.arange(1000000, chunks=1000)<\/p>\n
                    \u6267\u884c\u5e76\u884c\u8ba1\u7b97<\/strong><\/h2>\n
                    result = array.sum().compute()<\/p>\n
                    print(result)<\/p>\n
                    <\/code><\/pre>\n<\/p>\n
                    \u4e94\u3001DASK\u4e0e\u5176\u4ed6\u5206\u5e03\u5f0f\u8ba1\u7b97\u6846\u67b6\u6bd4\u8f83<\/p>\n<\/p>\n
                      \n
                    1. Dask vs Spark<\/strong><\/p>\n
                      Dask\u548cSpark\u90fd\u662f\u6d41\u884c\u7684\u5206\u5e03\u5f0f\u8ba1\u7b97\u6846\u67b6\uff0c\u5404\u6709\u4f18\u52a3\uff1a<\/li>\n<\/p>\n<\/ol>\n