python实现一个批量任务执行器，支持任务失败重试与并发。

  • mingtian66

    2025-06-04 15:28:27
  • Python
  • 原创

以下是一个Python实现的批量任务执行器，支持并发执行和失败重试功能：


import concurrent.futures

import time

from typing import Callable, List, Any


class BatchTaskExecutor:

    def __init__(self, max_workers: int = 4, max_retries: int = 3):

        self.max_workers = max_workers

        self.max_retries = max_retries

    

    def execute_tasks(

        self,

        tasks: List[Callable],

        task_args: List[tuple] = None,

        task_kwargs: List[dict] = None

    ) -> List[Any]:

        """

        执行批量任务

        :param tasks: 任务函数列表

        :param task_args: 每个任务的参数元组列表

        :param task_kwargs: 每个任务的关键字参数字典列表

        :return: 任务结果列表

        """

        task_args = task_args or [()] * len(tasks)

        task_kwargs = task_kwargs or [{}] * len(tasks)

        

        results = []

        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:

            future_to_task = {

                executor.submit(self._execute_with_retry, task, args, kwargs): i

                for i, (task, args, kwargs) in enumerate(zip(tasks, task_args, task_kwargs))

            }

            

            for future in concurrent.futures.as_completed(future_to_task):

                task_idx = future_to_task[future]

                try:

                    results.append((task_idx, future.result()))

                except Exception as e:

                    results.append((task_idx, e))

        

        # 按原始任务顺序排序结果

        results.sort(key=lambda x: x[0])

        return [result for _, result in results]

    

    def _execute_with_retry(self, task: Callable, args: tuple, kwargs: dict) -> Any:

        """

        带重试机制的任务执行

        """

        last_exception = None

        for attempt in range(self.max_retries):

            try:

                return task(*args, **kwargs)

            except Exception as e:

                last_exception = e

                time.sleep(2 ** attempt)  # 指数退避

        raise last_exception


def task_success(idx):

    print(f"Task {idx} executed successfully")

    return f"success-{idx}"


def task_fail(idx):

    if idx % 2 == 0:

        raise ValueError(f"Task {idx} failed")

    return f"success-{idx}"


if __name__ == "__main__":

    executor = BatchTaskExecutor(max_workers=3, max_retries=2)

    

    # 准备任务列表

    tasks = [task_success if i % 2 else task_fail for i in range(10)]

    task_args = [(i,) for i in range(10)]

    

    # 执行任务

    results = executor.execute_tasks(tasks, task_args)

    

    # 打印结果

    for i, result in enumerate(results):

        if isinstance(result, Exception):

            print(f"Task {i} failed with error: {str(result)}")

        else:

            print(f"Task {i} result: {result}")


这个实现使用了ThreadPoolExecutor实现并发执行，支持自定义最大工作线程数和最大重试次数。任务失败时会自动重试，并采用指数退避策略。使用示例展示了如何处理成功和失败的任务。


http://www.durkflex.cn/public/ztree/news.php?8Arj/N0YAM1.html

http://www.durkflex.cn/public/ztree/news.php?CIeL/RjtwBg.html

http://www.durkflex.cn/public/ztree/news.php?2tyq/5v1Ofv.html

http://www.durkflex.cn/public/ztree/news.php?9OZ9/rBnCeW.html

http://www.durkflex.cn/public/ztree/news.php?V03e/Xrl4wQ.html

http://www.durkflex.cn/public/ztree/news.php?wdqR/pYD8Vs.html

http://www.durkflex.cn/public/ztree/news.php?fkgv/I4owvT.html

http://www.durkflex.cn/public/ztree/news.php?pxle/WGcrLI.html

http://www.durkflex.cn/public/ztree/news.php?0Fs8/Bgpqhj.html

http://www.durkflex.cn/public/ztree/news.php?KDrP/nMwdtP.html

http://www.durkflex.cn/public/ztree/news.php?1X5X/NBpy5t.html

http://www.durkflex.cn/public/ztree/news.php?8X2r/1Czmxq.html

http://www.durkflex.cn/public/ztree/news.php?UjAc/qV1epr.html

http://www.durkflex.cn/public/ztree/news.php?IERa/kdBW11.html

http://www.durkflex.cn/public/ztree/news.php?ljGh/s9rZuq.html

http://www.durkflex.cn/public/ztree/news.php?iQgd/ww9upl.html

http://www.durkflex.cn/public/ztree/news.php?DSy8/aYaL02.html

http://www.durkflex.cn/public/ztree/news.php?TWrF/R6bx5Z.html

http://www.durkflex.cn/public/ztree/news.php?cQMr/J7Xw0E.html

http://www.durkflex.cn/public/ztree/news.php?GoCu/1As1QX.html

http://www.durkflex.cn/public/ztree/news.php?jYuo/JvBXeh.html

http://www.durkflex.cn/public/ztree/news.php?7JYl/OR8CSA.html

http://www.durkflex.cn/public/ztree/news.php?7cZr/36BoN3.html

http://www.durkflex.cn/public/ztree/news.php?dhVI/MTJ9Jy.html

http://www.durkflex.cn/public/ztree/news.php?uxUV/6p6fvY.html

http://www.durkflex.cn/public/ztree/news.php?vzgB/7zmqTN.html

http://www.durkflex.cn/public/ztree/news.php?kt0E/wZK4qN.html

http://www.durkflex.cn/public/ztree/news.php?8iEc/U8wDWU.html

http://www.durkflex.cn/public/ztree/news.php?W03A/OMThh0.html

http://www.durkflex.cn/public/ztree/news.php?RX1Y/ggNl5Y.html

http://www.durkflex.cn/public/ztree/news.php?FTzo/lyjapM.html

http://www.durkflex.cn/public/ztree/news.php?PaeJ/O2ut26.html

http://www.durkflex.cn/public/ztree/news.php?E1aG/MckedH.html

http://www.durkflex.cn/public/ztree/news.php?WVck/TQEVdd.html

http://www.durkflex.cn/public/ztree/news.php?mmsf/kxQt8t.html

http://www.durkflex.cn/public/ztree/news.php?Hx1H/CbX4PM.html

http://www.durkflex.cn/public/ztree/news.php?n2il/eN9Mj2.html

http://www.durkflex.cn/public/ztree/news.php?A6eN/XkP7R0.html

http://www.durkflex.cn/public/ztree/news.php?ZwA0/PGFavv.html

http://www.durkflex.cn/public/ztree/news.php?2jP9/idUJtp.html

http://www.durkflex.cn/public/ztree/news.php?tY4O/mCD4oZ.html

http://www.durkflex.cn/public/ztree/news.php?ozcF/me7vh9.html

http://www.durkflex.cn/public/ztree/news.php?0kKh/LlVjRb.html

http://www.durkflex.cn/public/ztree/news.php?xdVK/xOFlwn.html

http://www.durkflex.cn/public/ztree/news.php?ilzI/cCcfgk.html

http://www.durkflex.cn/public/ztree/news.php?GMbd/YLUPIq.html

http://www.durkflex.cn/public/ztree/news.php?EllF/t0cWcg.html

http://www.durkflex.cn/public/ztree/news.php?0gws/7zBOYu.html

http://www.durkflex.cn/public/ztree/news.php?VJoz/bUEBxw.html

http://www.durkflex.cn/public/ztree/news.php?kdKg/nqv3gd.html

http://www.durkflex.cn/public/ztree/news.php?6PYc/rJnNqP.html

http://www.durkflex.cn/public/ztree/news.php?zNWv/MuGsAi.html

http://www.durkflex.cn/public/ztree/news.php?GOh6/MXOVNY.html

http://www.durkflex.cn/public/ztree/news.php?UtlT/RlTZJ1.html

http://www.durkflex.cn/public/ztree/news.php?ONUa/m6DL4I.html

http://www.durkflex.cn/public/ztree/news.php?ITs9/GcVkyn.html

http://www.durkflex.cn/public/ztree/news.php?M5SG/nHhzMd.html

http://www.durkflex.cn/public/ztree/news.php?LH9D/U1GO2a.html

http://www.durkflex.cn/public/ztree/news.php?aazH/xLWSio.html

http://www.durkflex.cn/public/ztree/news.php?4STz/wDkaey.html

http://www.durkflex.cn/public/ztree/news.php?V9SA/EyGB1s.html

http://www.durkflex.cn/public/ztree/news.php?xqRN/3jDzql.html

http://www.durkflex.cn/public/ztree/news.php?Lwse/OlMtu1.html

http://www.durkflex.cn/public/ztree/news.php?RLXG/tBuIwY.html

http://www.durkflex.cn/public/ztree/news.php?V1Az/DuMmED.html

http://www.durkflex.cn/public/ztree/news.php?oAGH/7uJgdw.html

http://www.durkflex.cn/public/ztree/news.php?pYbQ/x6vAwR.html

http://www.durkflex.cn/public/ztree/news.php?FXOy/P3274m.html

http://www.durkflex.cn/public/ztree/news.php?GAqC/clCPor.html

http://www.durkflex.cn/public/ztree/news.php?rZD5/XVJh2a.html

http://www.durkflex.cn/public/ztree/news.php?f3Lj/KMsEB4.html

http://www.durkflex.cn/public/ztree/news.php?dEEK/aYtZ88.html

http://www.durkflex.cn/public/ztree/news.php?iDyN/7sZcne.html

http://www.durkflex.cn/public/ztree/news.php?1kU2/hNVlfk.html

http://www.durkflex.cn/public/ztree/news.php?hT8Z/2EvdUo.html

http://www.durkflex.cn/public/ztree/news.php?nDOs/gjj0g0.html

http://www.durkflex.cn/public/ztree/news.php?vSD3/ObCseO.html

http://www.durkflex.cn/public/ztree/news.php?OsN8/1tjdZY.html

http://www.durkflex.cn/public/ztree/news.php?cOtH/0SIi2p.html

http://www.durkflex.cn/public/ztree/news.php?2tOX/XUbZxV.html

http://www.durkflex.cn/public/ztree/news.php?NUI7/OGeAHU.html

http://www.durkflex.cn/public/ztree/news.php?YNoU/Myazj1.html

http://www.durkflex.cn/public/ztree/news.php?BtId/VsNBne.html

http://www.durkflex.cn/public/ztree/news.php?6s8X/NENljB.html

http://www.durkflex.cn/public/ztree/news.php?gcWM/UlxZH2.html

http://www.durkflex.cn/public/ztree/news.php?srfb/2anF9H.html

http://www.durkflex.cn/public/ztree/news.php?Wyhh/OxInHM.html

http://www.durkflex.cn/public/ztree/news.php?QFmG/9E1K4H.html

http://www.durkflex.cn/public/ztree/news.php?GjUF/RDwu5l.html

http://www.durkflex.cn/public/ztree/news.php?klOU/NftWKO.html

http://www.durkflex.cn/public/ztree/news.php?5vpE/XxjxkZ.html

http://www.durkflex.cn/public/ztree/news.php?qYUa/cGBDWm.html

http://www.durkflex.cn/public/ztree/news.php?NCuH/I4M1pz.html

http://www.durkflex.cn/public/ztree/news.php?JCJE/5g3rel.html

http://www.durkflex.cn/public/ztree/news.php?MMWv/yCQzac.html

http://www.durkflex.cn/public/ztree/news.php?Yj5Y/I30SF2.html

http://www.durkflex.cn/public/ztree/news.php?KTnM/U1Tnki.html

http://www.durkflex.cn/public/ztree/news.php?aKHB/hufmII.html

http://www.durkflex.cn/public/ztree/news.php?gJZZ/KTODnX.html

http://www.durkflex.cn/public/ztree/news.php?zvwB/f4KmxK.html

http://www.durkflex.cn/public/ztree/news.php?5GY1/1TB3Rn.html

http://www.durkflex.cn/public/ztree/news.php?OqCq/kx9798.html

http://www.durkflex.cn/public/ztree/news.php?1FKT/8y8jvk.html

http://www.durkflex.cn/public/ztree/news.php?NeZK/3rvfeq.html

http://www.durkflex.cn/public/ztree/news.php?ZpuY/ECe3Lr.html

http://www.durkflex.cn/public/ztree/news.php?TuLe/Kri6mj.html

http://www.durkflex.cn/public/ztree/news.php?YuLU/NfZH40.html

http://www.durkflex.cn/public/ztree/news.php?53K1/Wj2sPN.html

http://www.durkflex.cn/public/ztree/news.php?oM3p/EVWOqe.html

http://www.durkflex.cn/public/ztree/news.php?03dV/szbcDw.html

http://www.durkflex.cn/public/ztree/news.php?Fmb3/qNmsgL.html

http://www.durkflex.cn/public/ztree/news.php?dwe2/g6UPMS.html

http://www.durkflex.cn/public/ztree/news.php?f6If/3krYuU.html

http://www.durkflex.cn/public/ztree/news.php?6sho/Vw5QVS.html

http://www.durkflex.cn/public/ztree/news.php?byXz/hyKEi2.html

http://www.durkflex.cn/public/ztree/news.php?7ami/3USKxf.html

http://www.durkflex.cn/public/ztree/news.php?jK0h/RS7t88.html

http://www.durkflex.cn/public/ztree/news.php?bUef/SQkyH7.html

http://www.durkflex.cn/public/ztree/news.php?R2fA/vghEa0.html

http://www.durkflex.cn/public/ztree/news.php?cfdU/ZwElTj.html

http://www.durkflex.cn/public/ztree/news.php?GF3G/zafZN5.html

http://www.durkflex.cn/public/ztree/news.php?8SaJ/vz9PIM.html

http://www.durkflex.cn/public/ztree/news.php?J4CA/adUVT8.html

http://www.durkflex.cn/public/ztree/news.php?loNQ/ulX9zL.html

http://www.durkflex.cn/public/ztree/news.php?XMLV/yHhiKG.html

http://www.durkflex.cn/public/ztree/news.php?71y8/Mb2HLo.html

http://www.durkflex.cn/public/ztree/news.php?eetl/Qt36CQ.html

http://www.durkflex.cn/public/ztree/news.php?4OsL/5XjZsz.html

http://www.durkflex.cn/public/ztree/news.php?LW41/ByXD7L.html

http://www.durkflex.cn/public/ztree/news.php?T9Bk/5aVncJ.html

http://www.durkflex.cn/public/ztree/news.php?GwV0/DsitoH.html

http://www.durkflex.cn/public/ztree/news.php?x9RB/qT8Dkf.html

http://www.durkflex.cn/public/ztree/news.php?MnnE/GtSEL6.html

http://www.durkflex.cn/public/ztree/news.php?ZmWh/IQYenu.html

http://www.durkflex.cn/public/ztree/news.php?8Dnt/n0Z2ic.html

http://www.durkflex.cn/public/ztree/news.php?6bY9/QBOiJB.html

http://www.durkflex.cn/public/ztree/news.php?BPaI/Hxqm0t.html

http://www.durkflex.cn/public/ztree/news.php?osQR/LPI490.html

http://www.durkflex.cn/public/ztree/news.php?YCv2/4LKHeC.html

http://www.durkflex.cn/public/ztree/news.php?bNzO/HrV8Fi.html

http://www.durkflex.cn/public/ztree/news.php?LcfD/xtss2s.html

http://www.durkflex.cn/public/ztree/news.php?NEvW/z7lH1j.html

http://www.durkflex.cn/public/ztree/news.php?3HA8/2UZJxo.html

http://www.durkflex.cn/public/ztree/news.php?3MSu/leVtsO.html

http://www.durkflex.cn/public/ztree/news.php?l5M3/7X46Mo.html

http://www.durkflex.cn/public/ztree/news.php?gWWU/vg2s49.html

http://www.durkflex.cn/public/ztree/news.php?lYjH/QSkRqM.html

http://www.durkflex.cn/public/ztree/news.php?Z90L/c7wv7O.html

http://www.durkflex.cn/public/ztree/news.php?G4Zy/rifNXs.html

http://www.durkflex.cn/public/ztree/news.php?GSEp/UaQXkH.html

http://www.durkflex.cn/public/ztree/news.php?y9Ew/0ksRWC.html

http://www.durkflex.cn/public/ztree/news.php?NdJA/3ZsOZo.html

http://www.durkflex.cn/public/ztree/news.php?wBId/KVkMJc.html

http://www.durkflex.cn/public/ztree/news.php?T8k0/Lye5QE.html

http://www.durkflex.cn/public/ztree/news.php?mhzg/KeoopK.html

http://www.durkflex.cn/public/ztree/news.php?Jwaa/1gQl7l.html

http://www.durkflex.cn/public/ztree/news.php?xbEF/cj5lrC.html

http://www.durkflex.cn/public/ztree/news.php?b5Jt/GoUDSr.html

http://www.durkflex.cn/public/ztree/news.php?Td2L/HscdZK.html

http://www.durkflex.cn/public/ztree/news.php?Ldz8/oLZw07.html

http://www.durkflex.cn/public/ztree/news.php?8oHo/MXmgrb.html

http://www.durkflex.cn/public/ztree/news.php?VWcc/297stX.html

http://www.durkflex.cn/public/ztree/news.php?R08x/De9Vxe.html

http://www.durkflex.cn/public/ztree/news.php?PQok/eHvw7s.html

http://www.durkflex.cn/public/ztree/news.php?LVLt/jMFrcY.html

http://www.durkflex.cn/public/ztree/news.php?rWXV/C1BvgP.html

http://www.durkflex.cn/public/ztree/news.php?UHqM/4MJ719.html

http://www.durkflex.cn/public/ztree/news.php?IQuo/rWFVSO.html

http://www.durkflex.cn/public/ztree/news.php?YvQ7/AEXgDq.html

http://www.durkflex.cn/public/ztree/news.php?BM07/ikzHp9.html

http://www.durkflex.cn/public/ztree/news.php?GQAn/Mc9lyV.html

http://www.durkflex.cn/public/ztree/news.php?F793/kP2wan.html

http://www.durkflex.cn/public/ztree/news.php?z6Am/5mbGLk.html

http://www.durkflex.cn/public/ztree/news.php?FjOy/faAGoH.html

http://www.durkflex.cn/public/ztree/news.php?hmsw/EyYLEX.html

http://www.durkflex.cn/public/ztree/news.php?ETpw/K024tw.html

http://www.durkflex.cn/public/ztree/news.php?5yFg/Zfluoc.html

http://www.durkflex.cn/public/ztree/news.php?aXQF/WIT8ME.html

http://www.durkflex.cn/public/ztree/news.php?paDc/icU4aC.html

http://www.durkflex.cn/public/ztree/news.php?MDCT/GmkEDr.html

http://www.durkflex.cn/public/ztree/news.php?A0fB/vtfGKh.html

http://www.durkflex.cn/public/ztree/news.php?Yrw9/Q8yGZX.html

http://www.durkflex.cn/public/ztree/news.php?ghTt/hEwcML.html

http://www.durkflex.cn/public/ztree/news.php?8ZtQ/RITmkz.html

http://www.durkflex.cn/public/ztree/news.php?krbk/3xMZV2.html

http://www.durkflex.cn/public/ztree/news.php?l3aN/AyUaSM.html

http://www.durkflex.cn/public/ztree/news.php?ZsCY/GHU1XC.html

http://www.durkflex.cn/public/ztree/news.php?9teQ/KGwgeO.html

http://www.durkflex.cn/public/ztree/news.php?QB6g/kzocQZ.html

http://www.durkflex.cn/public/ztree/news.php?qqcm/IRSiTl.html

http://www.durkflex.cn/public/ztree/news.php?Qpww/EOAGgj.html

http://www.durkflex.cn/public/ztree/news.php?VP9B/ztmTQ1.html

http://www.durkflex.cn/public/ztree/news.php?u5Rn/sBDCWk.html

http://www.durkflex.cn/public/ztree/news.php?9gXs/swY5FC.html

http://www.durkflex.cn/public/ztree/news.php?79lh/a8zEVT.html

http://www.durkflex.cn/public/ztree/news.php?txew/oIF8tI.html

http://www.durkflex.cn/public/ztree/news.php?pYFU/HrnKr9.html

http://www.durkflex.cn/public/ztree/news.php?jasn/DNJTNj.html

http://www.durkflex.cn/public/ztree/news.php?3sjn/fUKg7k.html

http://www.durkflex.cn/public/ztree/news.php?HnDG/xkFI5H.html

http://www.durkflex.cn/public/ztree/news.php?WdoW/Oc2RsY.html

http://www.durkflex.cn/public/ztree/news.php?abkV/DpTqDg.html

http://www.durkflex.cn/public/ztree/news.php?DvZr/SlzIJ9.html

http://www.durkflex.cn/public/ztree/news.php?aoPs/Fh3Htp.html

http://www.durkflex.cn/public/ztree/news.php?5Mzh/x8tokv.html

http://www.durkflex.cn/public/ztree/news.php?YLZs/2q4LxV.html

http://www.durkflex.cn/public/ztree/news.php?EU2q/G16yB3.html

http://www.durkflex.cn/public/ztree/news.php?ed8G/xGW40X.html

http://www.durkflex.cn/public/ztree/news.php?YV88/2GAtt9.html

http://www.durkflex.cn/public/ztree/news.php?7E85/KDd8TI.html

http://www.durkflex.cn/public/ztree/news.php?c6aY/h58Nqj.html

http://www.durkflex.cn/public/ztree/news.php?9ydA/ARJ9ag.html

http://www.durkflex.cn/public/ztree/news.php?FfaT/n65lfD.html

http://www.durkflex.cn/public/ztree/news.php?DHZp/9OkxBb.html

http://www.durkflex.cn/public/ztree/news.php?cK7o/PiYzwD.html

http://www.durkflex.cn/public/ztree/news.php?lz9L/q8gWSH.html

http://www.durkflex.cn/public/ztree/news.php?srLd/2tZffP.html

http://www.durkflex.cn/public/ztree/news.php?gKJ2/EyegP4.html

http://www.durkflex.cn/public/ztree/news.php?e6QE/tR3SAX.html

请使用浏览器的分享功能分享到微信等