python写一个多进程数据管道，支持统计 CPU 和内存使用

  • mingtian66

    2025-06-04 15:29:19
  • Python
  • 原创


import multiprocessing

import time

import psutil

import os

from queue import Empty


class ProcessMonitor(multiprocessing.Process):

    def __init__(self, stats_queue, interval=1):

        super().__init__()

        self.stats_queue = stats_queue

        self.interval = interval

        self.stop_event = multiprocessing.Event()

        

    def run(self):

        while not self.stop_event.is_set():

            cpu_percent = psutil.cpu_percent(interval=self.interval)

            mem_info = psutil.virtual_memory()

            self.stats_queue.put({

                'cpu': cpu_percent,

                'memory': mem_info.percent,

                'timestamp': time.time()

            })

    

    def stop(self):

        self.stop_event.set()


class WorkerProcess(multiprocessing.Process):

    def __init__(self, task_queue, result_queue, worker_id):

        super().__init__()

        self.task_queue = task_queue

        self.result_queue = result_queue

        self.worker_id = worker_id

        

    def run(self):

        while True:

            try:

                task = self.task_queue.get(timeout=1)

                if task is None:  # 终止信号

                    break

                    

                # 模拟任务处理

                result = f"Processed {task} by worker {self.worker_id}"

                self.result_queue.put(result)

                

            except Empty:

                continue


class MultiProcessPipeline:

    def __init__(self, num_workers=4):

        self.num_workers = num_workers

        self.task_queue = multiprocessing.Queue()

        self.result_queue = multiprocessing.Queue()

        self.stats_queue = multiprocessing.Queue()

        self.workers = []

        self.monitor = ProcessMonitor(self.stats_queue)

        

    def start(self):

        # 启动监控进程

        self.monitor.start()

        

        # 启动工作进程

        for i in range(self.num_workers):

            worker = WorkerProcess(self.task_queue, self.result_queue, i)

            worker.start()

            self.workers.append(worker)

    

    def add_task(self, task):

        self.task_queue.put(task)

    

    def get_results(self):

        results = []

        while not self.result_queue.empty():

            results.append(self.result_queue.get())

        return results

    

    def get_stats(self):

        stats = []

        while not self.stats_queue.empty():

            stats.append(self.stats_queue.get())

        return stats

    

    def stop(self):

        # 发送终止信号给工作进程

        for _ in range(self.num_workers):

            self.task_queue.put(None)

            

        # 等待工作进程结束

        for worker in self.workers:

            worker.join()

            

        # 停止监控进程

        self.monitor.stop()

        self.monitor.join()


if __name__ == "__main__":

    pipeline = MultiProcessPipeline(num_workers=2)

    pipeline.start()

    

    # 添加任务

    for i in range(10):

        pipeline.add_task(f"Task-{i}")

    

    # 等待任务完成

    time.sleep(3)

    

    # 获取结果

    print("Results:", pipeline.get_results())

    

    # 获取统计信息

    stats = pipeline.get_stats()

    print("CPU and Memory Stats:")

    for stat in stats:

        print(f"Time: {stat['timestamp']}, CPU: {stat['cpu']}%, Memory: {stat['memory']}%")

    

    pipeline.stop()

这个多进程数据管道实现包含任务队列、结果队列和统计队列，使用ProcessMonitor监控CPU和内存使用情况，WorkerProcess处理任务，MultiProcessPipeline管理整个流程。

http://www.durkflex.cn/public/ztree/news.php?8Arj/N0YAM1.html

http://www.durkflex.cn/public/ztree/news.php?CIeL/RjtwBg.html

http://www.durkflex.cn/public/ztree/news.php?2tyq/5v1Ofv.html

http://www.durkflex.cn/public/ztree/news.php?9OZ9/rBnCeW.html

http://www.durkflex.cn/public/ztree/news.php?V03e/Xrl4wQ.html

http://www.durkflex.cn/public/ztree/news.php?wdqR/pYD8Vs.html

http://www.durkflex.cn/public/ztree/news.php?fkgv/I4owvT.html

http://www.durkflex.cn/public/ztree/news.php?pxle/WGcrLI.html

http://www.durkflex.cn/public/ztree/news.php?0Fs8/Bgpqhj.html

http://www.durkflex.cn/public/ztree/news.php?KDrP/nMwdtP.html

http://www.durkflex.cn/public/ztree/news.php?1X5X/NBpy5t.html

http://www.durkflex.cn/public/ztree/news.php?8X2r/1Czmxq.html

http://www.durkflex.cn/public/ztree/news.php?UjAc/qV1epr.html

http://www.durkflex.cn/public/ztree/news.php?IERa/kdBW11.html

http://www.durkflex.cn/public/ztree/news.php?ljGh/s9rZuq.html

http://www.durkflex.cn/public/ztree/news.php?iQgd/ww9upl.html

http://www.durkflex.cn/public/ztree/news.php?DSy8/aYaL02.html

http://www.durkflex.cn/public/ztree/news.php?TWrF/R6bx5Z.html

http://www.durkflex.cn/public/ztree/news.php?cQMr/J7Xw0E.html

http://www.durkflex.cn/public/ztree/news.php?GoCu/1As1QX.html

http://www.durkflex.cn/public/ztree/news.php?jYuo/JvBXeh.html

http://www.durkflex.cn/public/ztree/news.php?7JYl/OR8CSA.html

http://www.durkflex.cn/public/ztree/news.php?7cZr/36BoN3.html

http://www.durkflex.cn/public/ztree/news.php?dhVI/MTJ9Jy.html

http://www.durkflex.cn/public/ztree/news.php?uxUV/6p6fvY.html

http://www.durkflex.cn/public/ztree/news.php?vzgB/7zmqTN.html

http://www.durkflex.cn/public/ztree/news.php?kt0E/wZK4qN.html

http://www.durkflex.cn/public/ztree/news.php?8iEc/U8wDWU.html

http://www.durkflex.cn/public/ztree/news.php?W03A/OMThh0.html

http://www.durkflex.cn/public/ztree/news.php?RX1Y/ggNl5Y.html

http://www.durkflex.cn/public/ztree/news.php?FTzo/lyjapM.html

http://www.durkflex.cn/public/ztree/news.php?PaeJ/O2ut26.html

http://www.durkflex.cn/public/ztree/news.php?E1aG/MckedH.html

http://www.durkflex.cn/public/ztree/news.php?WVck/TQEVdd.html

http://www.durkflex.cn/public/ztree/news.php?mmsf/kxQt8t.html

http://www.durkflex.cn/public/ztree/news.php?Hx1H/CbX4PM.html

http://www.durkflex.cn/public/ztree/news.php?n2il/eN9Mj2.html

http://www.durkflex.cn/public/ztree/news.php?A6eN/XkP7R0.html

http://www.durkflex.cn/public/ztree/news.php?ZwA0/PGFavv.html

http://www.durkflex.cn/public/ztree/news.php?2jP9/idUJtp.html

http://www.durkflex.cn/public/ztree/news.php?tY4O/mCD4oZ.html

http://www.durkflex.cn/public/ztree/news.php?ozcF/me7vh9.html

http://www.durkflex.cn/public/ztree/news.php?0kKh/LlVjRb.html

http://www.durkflex.cn/public/ztree/news.php?xdVK/xOFlwn.html

http://www.durkflex.cn/public/ztree/news.php?ilzI/cCcfgk.html

http://www.durkflex.cn/public/ztree/news.php?GMbd/YLUPIq.html

http://www.durkflex.cn/public/ztree/news.php?EllF/t0cWcg.html

http://www.durkflex.cn/public/ztree/news.php?0gws/7zBOYu.html

http://www.durkflex.cn/public/ztree/news.php?VJoz/bUEBxw.html

http://www.durkflex.cn/public/ztree/news.php?kdKg/nqv3gd.html

http://www.durkflex.cn/public/ztree/news.php?6PYc/rJnNqP.html

http://www.durkflex.cn/public/ztree/news.php?zNWv/MuGsAi.html

http://www.durkflex.cn/public/ztree/news.php?GOh6/MXOVNY.html

http://www.durkflex.cn/public/ztree/news.php?UtlT/RlTZJ1.html

http://www.durkflex.cn/public/ztree/news.php?ONUa/m6DL4I.html

http://www.durkflex.cn/public/ztree/news.php?ITs9/GcVkyn.html

http://www.durkflex.cn/public/ztree/news.php?M5SG/nHhzMd.html

http://www.durkflex.cn/public/ztree/news.php?LH9D/U1GO2a.html

http://www.durkflex.cn/public/ztree/news.php?aazH/xLWSio.html

http://www.durkflex.cn/public/ztree/news.php?4STz/wDkaey.html

http://www.durkflex.cn/public/ztree/news.php?V9SA/EyGB1s.html

http://www.durkflex.cn/public/ztree/news.php?xqRN/3jDzql.html

http://www.durkflex.cn/public/ztree/news.php?Lwse/OlMtu1.html

http://www.durkflex.cn/public/ztree/news.php?RLXG/tBuIwY.html

http://www.durkflex.cn/public/ztree/news.php?V1Az/DuMmED.html

http://www.durkflex.cn/public/ztree/news.php?oAGH/7uJgdw.html

http://www.durkflex.cn/public/ztree/news.php?pYbQ/x6vAwR.html

http://www.durkflex.cn/public/ztree/news.php?FXOy/P3274m.html

http://www.durkflex.cn/public/ztree/news.php?GAqC/clCPor.html

http://www.durkflex.cn/public/ztree/news.php?rZD5/XVJh2a.html

http://www.durkflex.cn/public/ztree/news.php?f3Lj/KMsEB4.html

http://www.durkflex.cn/public/ztree/news.php?dEEK/aYtZ88.html

http://www.durkflex.cn/public/ztree/news.php?iDyN/7sZcne.html

http://www.durkflex.cn/public/ztree/news.php?1kU2/hNVlfk.html

http://www.durkflex.cn/public/ztree/news.php?hT8Z/2EvdUo.html

http://www.durkflex.cn/public/ztree/news.php?nDOs/gjj0g0.html

http://www.durkflex.cn/public/ztree/news.php?vSD3/ObCseO.html

http://www.durkflex.cn/public/ztree/news.php?OsN8/1tjdZY.html

http://www.durkflex.cn/public/ztree/news.php?cOtH/0SIi2p.html

http://www.durkflex.cn/public/ztree/news.php?2tOX/XUbZxV.html

http://www.durkflex.cn/public/ztree/news.php?NUI7/OGeAHU.html

http://www.durkflex.cn/public/ztree/news.php?YNoU/Myazj1.html

http://www.durkflex.cn/public/ztree/news.php?BtId/VsNBne.html

http://www.durkflex.cn/public/ztree/news.php?6s8X/NENljB.html

http://www.durkflex.cn/public/ztree/news.php?gcWM/UlxZH2.html

http://www.durkflex.cn/public/ztree/news.php?srfb/2anF9H.html

http://www.durkflex.cn/public/ztree/news.php?Wyhh/OxInHM.html

http://www.durkflex.cn/public/ztree/news.php?QFmG/9E1K4H.html

http://www.durkflex.cn/public/ztree/news.php?GjUF/RDwu5l.html

http://www.durkflex.cn/public/ztree/news.php?klOU/NftWKO.html

http://www.durkflex.cn/public/ztree/news.php?5vpE/XxjxkZ.html

http://www.durkflex.cn/public/ztree/news.php?qYUa/cGBDWm.html

http://www.durkflex.cn/public/ztree/news.php?NCuH/I4M1pz.html

http://www.durkflex.cn/public/ztree/news.php?JCJE/5g3rel.html

http://www.durkflex.cn/public/ztree/news.php?MMWv/yCQzac.html

http://www.durkflex.cn/public/ztree/news.php?Yj5Y/I30SF2.html

http://www.durkflex.cn/public/ztree/news.php?KTnM/U1Tnki.html

http://www.durkflex.cn/public/ztree/news.php?aKHB/hufmII.html

http://www.durkflex.cn/public/ztree/news.php?gJZZ/KTODnX.html

http://www.durkflex.cn/public/ztree/news.php?zvwB/f4KmxK.html

http://www.durkflex.cn/public/ztree/news.php?5GY1/1TB3Rn.html

http://www.durkflex.cn/public/ztree/news.php?OqCq/kx9798.html

http://www.durkflex.cn/public/ztree/news.php?1FKT/8y8jvk.html

http://www.durkflex.cn/public/ztree/news.php?NeZK/3rvfeq.html

http://www.durkflex.cn/public/ztree/news.php?ZpuY/ECe3Lr.html

http://www.durkflex.cn/public/ztree/news.php?TuLe/Kri6mj.html

http://www.durkflex.cn/public/ztree/news.php?YuLU/NfZH40.html

http://www.durkflex.cn/public/ztree/news.php?53K1/Wj2sPN.html

http://www.durkflex.cn/public/ztree/news.php?oM3p/EVWOqe.html

http://www.durkflex.cn/public/ztree/news.php?03dV/szbcDw.html

http://www.durkflex.cn/public/ztree/news.php?Fmb3/qNmsgL.html

http://www.durkflex.cn/public/ztree/news.php?dwe2/g6UPMS.html

http://www.durkflex.cn/public/ztree/news.php?f6If/3krYuU.html

http://www.durkflex.cn/public/ztree/news.php?6sho/Vw5QVS.html

http://www.durkflex.cn/public/ztree/news.php?byXz/hyKEi2.html

http://www.durkflex.cn/public/ztree/news.php?7ami/3USKxf.html

http://www.durkflex.cn/public/ztree/news.php?jK0h/RS7t88.html

http://www.durkflex.cn/public/ztree/news.php?bUef/SQkyH7.html

http://www.durkflex.cn/public/ztree/news.php?R2fA/vghEa0.html

http://www.durkflex.cn/public/ztree/news.php?cfdU/ZwElTj.html

http://www.durkflex.cn/public/ztree/news.php?GF3G/zafZN5.html

http://www.durkflex.cn/public/ztree/news.php?8SaJ/vz9PIM.html

http://www.durkflex.cn/public/ztree/news.php?J4CA/adUVT8.html

http://www.durkflex.cn/public/ztree/news.php?loNQ/ulX9zL.html

http://www.durkflex.cn/public/ztree/news.php?XMLV/yHhiKG.html

http://www.durkflex.cn/public/ztree/news.php?71y8/Mb2HLo.html

http://www.durkflex.cn/public/ztree/news.php?eetl/Qt36CQ.html

http://www.durkflex.cn/public/ztree/news.php?4OsL/5XjZsz.html

http://www.durkflex.cn/public/ztree/news.php?LW41/ByXD7L.html

http://www.durkflex.cn/public/ztree/news.php?T9Bk/5aVncJ.html

http://www.durkflex.cn/public/ztree/news.php?GwV0/DsitoH.html

http://www.durkflex.cn/public/ztree/news.php?x9RB/qT8Dkf.html

http://www.durkflex.cn/public/ztree/news.php?MnnE/GtSEL6.html

http://www.durkflex.cn/public/ztree/news.php?ZmWh/IQYenu.html

http://www.durkflex.cn/public/ztree/news.php?8Dnt/n0Z2ic.html

http://www.durkflex.cn/public/ztree/news.php?6bY9/QBOiJB.html

http://www.durkflex.cn/public/ztree/news.php?BPaI/Hxqm0t.html

http://www.durkflex.cn/public/ztree/news.php?osQR/LPI490.html

http://www.durkflex.cn/public/ztree/news.php?YCv2/4LKHeC.html

http://www.durkflex.cn/public/ztree/news.php?bNzO/HrV8Fi.html

http://www.durkflex.cn/public/ztree/news.php?LcfD/xtss2s.html

http://www.durkflex.cn/public/ztree/news.php?NEvW/z7lH1j.html

http://www.durkflex.cn/public/ztree/news.php?3HA8/2UZJxo.html

http://www.durkflex.cn/public/ztree/news.php?3MSu/leVtsO.html

http://www.durkflex.cn/public/ztree/news.php?l5M3/7X46Mo.html

http://www.durkflex.cn/public/ztree/news.php?gWWU/vg2s49.html

http://www.durkflex.cn/public/ztree/news.php?lYjH/QSkRqM.html

http://www.durkflex.cn/public/ztree/news.php?Z90L/c7wv7O.html

http://www.durkflex.cn/public/ztree/news.php?G4Zy/rifNXs.html

http://www.durkflex.cn/public/ztree/news.php?GSEp/UaQXkH.html

http://www.durkflex.cn/public/ztree/news.php?y9Ew/0ksRWC.html

http://www.durkflex.cn/public/ztree/news.php?NdJA/3ZsOZo.html

http://www.durkflex.cn/public/ztree/news.php?wBId/KVkMJc.html

http://www.durkflex.cn/public/ztree/news.php?T8k0/Lye5QE.html

http://www.durkflex.cn/public/ztree/news.php?mhzg/KeoopK.html

http://www.durkflex.cn/public/ztree/news.php?Jwaa/1gQl7l.html

http://www.durkflex.cn/public/ztree/news.php?xbEF/cj5lrC.html

http://www.durkflex.cn/public/ztree/news.php?b5Jt/GoUDSr.html

http://www.durkflex.cn/public/ztree/news.php?Td2L/HscdZK.html

http://www.durkflex.cn/public/ztree/news.php?Ldz8/oLZw07.html

http://www.durkflex.cn/public/ztree/news.php?8oHo/MXmgrb.html

http://www.durkflex.cn/public/ztree/news.php?VWcc/297stX.html

http://www.durkflex.cn/public/ztree/news.php?R08x/De9Vxe.html

http://www.durkflex.cn/public/ztree/news.php?PQok/eHvw7s.html

http://www.durkflex.cn/public/ztree/news.php?LVLt/jMFrcY.html

http://www.durkflex.cn/public/ztree/news.php?rWXV/C1BvgP.html

http://www.durkflex.cn/public/ztree/news.php?UHqM/4MJ719.html

http://www.durkflex.cn/public/ztree/news.php?IQuo/rWFVSO.html

http://www.durkflex.cn/public/ztree/news.php?YvQ7/AEXgDq.html

http://www.durkflex.cn/public/ztree/news.php?BM07/ikzHp9.html

http://www.durkflex.cn/public/ztree/news.php?GQAn/Mc9lyV.html

http://www.durkflex.cn/public/ztree/news.php?F793/kP2wan.html

http://www.durkflex.cn/public/ztree/news.php?z6Am/5mbGLk.html

http://www.durkflex.cn/public/ztree/news.php?FjOy/faAGoH.html

http://www.durkflex.cn/public/ztree/news.php?hmsw/EyYLEX.html

http://www.durkflex.cn/public/ztree/news.php?ETpw/K024tw.html

http://www.durkflex.cn/public/ztree/news.php?5yFg/Zfluoc.html

http://www.durkflex.cn/public/ztree/news.php?aXQF/WIT8ME.html

http://www.durkflex.cn/public/ztree/news.php?paDc/icU4aC.html

http://www.durkflex.cn/public/ztree/news.php?MDCT/GmkEDr.html

http://www.durkflex.cn/public/ztree/news.php?A0fB/vtfGKh.html

http://www.durkflex.cn/public/ztree/news.php?Yrw9/Q8yGZX.html

http://www.durkflex.cn/public/ztree/news.php?ghTt/hEwcML.html

http://www.durkflex.cn/public/ztree/news.php?8ZtQ/RITmkz.html

http://www.durkflex.cn/public/ztree/news.php?krbk/3xMZV2.html

http://www.durkflex.cn/public/ztree/news.php?l3aN/AyUaSM.html

http://www.durkflex.cn/public/ztree/news.php?ZsCY/GHU1XC.html

http://www.durkflex.cn/public/ztree/news.php?9teQ/KGwgeO.html

http://www.durkflex.cn/public/ztree/news.php?QB6g/kzocQZ.html

http://www.durkflex.cn/public/ztree/news.php?qqcm/IRSiTl.html

http://www.durkflex.cn/public/ztree/news.php?Qpww/EOAGgj.html

http://www.durkflex.cn/public/ztree/news.php?VP9B/ztmTQ1.html

http://www.durkflex.cn/public/ztree/news.php?u5Rn/sBDCWk.html

http://www.durkflex.cn/public/ztree/news.php?9gXs/swY5FC.html

http://www.durkflex.cn/public/ztree/news.php?79lh/a8zEVT.html

http://www.durkflex.cn/public/ztree/news.php?txew/oIF8tI.html

http://www.durkflex.cn/public/ztree/news.php?pYFU/HrnKr9.html

http://www.durkflex.cn/public/ztree/news.php?jasn/DNJTNj.html

http://www.durkflex.cn/public/ztree/news.php?3sjn/fUKg7k.html

http://www.durkflex.cn/public/ztree/news.php?HnDG/xkFI5H.html

http://www.durkflex.cn/public/ztree/news.php?WdoW/Oc2RsY.html

http://www.durkflex.cn/public/ztree/news.php?abkV/DpTqDg.html

http://www.durkflex.cn/public/ztree/news.php?DvZr/SlzIJ9.html

http://www.durkflex.cn/public/ztree/news.php?aoPs/Fh3Htp.html

http://www.durkflex.cn/public/ztree/news.php?5Mzh/x8tokv.html

http://www.durkflex.cn/public/ztree/news.php?YLZs/2q4LxV.html

http://www.durkflex.cn/public/ztree/news.php?EU2q/G16yB3.html

http://www.durkflex.cn/public/ztree/news.php?ed8G/xGW40X.html

http://www.durkflex.cn/public/ztree/news.php?YV88/2GAtt9.html

http://www.durkflex.cn/public/ztree/news.php?7E85/KDd8TI.html

http://www.durkflex.cn/public/ztree/news.php?c6aY/h58Nqj.html

http://www.durkflex.cn/public/ztree/news.php?9ydA/ARJ9ag.html

http://www.durkflex.cn/public/ztree/news.php?FfaT/n65lfD.html

http://www.durkflex.cn/public/ztree/news.php?DHZp/9OkxBb.html

http://www.durkflex.cn/public/ztree/news.php?cK7o/PiYzwD.html

http://www.durkflex.cn/public/ztree/news.php?lz9L/q8gWSH.html

http://www.durkflex.cn/public/ztree/news.php?srLd/2tZffP.html

http://www.durkflex.cn/public/ztree/news.php?gKJ2/EyegP4.html

http://www.durkflex.cn/public/ztree/news.php?e6QE/tR3SAX.html

请使用浏览器的分享功能分享到微信等