jhdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z ddl mZmZmZmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!ddl"m#Z#Gd d Z$dS) z Dirty Arbiter Process Asyncio-based arbiter that manages the dirty worker pool and routes requests from HTTP workers to available dirty workers. N)util)get_app_workers_attributeparse_dirty_app_spec) DirtyErrorDirtyNoWorkersAvailableErrorDirtyTimeoutErrorDirtyWorkerError) DirtyProtocolmake_error_response make_response STASH_OP_PUT STASH_OP_GETSTASH_OP_DELETE STASH_OP_KEYSSTASH_OP_CLEAR STASH_OP_INFOSTASH_OP_ENSURESTASH_OP_DELETE_TABLESTASH_OP_TABLESSTASH_OP_EXISTS MANAGE_OP_ADDMANAGE_OP_REMOVE) DirtyWorkerceZdZdZddDZdZd(dZdZdZ d Z d Z d Z d Z d ZdZdZdZdZdZdZdZdZdZd)dZdZdZdZdZdZdZd*dZd Z d!Z!d"Z"d#Z#d$Z$d+d&Z%d'Z&dS), DirtyArbitera4 Dirty arbiter that manages the dirty worker pool. The arbiter runs an asyncio event loop and handles: - Spawning and managing dirty worker processes - Accepting connections from HTTP workers - Routing requests to available dirty workers - Monitoring worker health via heartbeat c>g|]}ttd|zS)zSIG%s)getattrsignal).0xs c/var/www/finuniver-perm.ru/html/student/venv/lib/python3.11/site-packages/gunicorn/dirty/arbiter.py zDirtyArbiter.<s8EEEwvw{++EEEz*HUP QUIT INT TERM TTIN TTOU USR1 USR2 CHLDNc2||_||_d|_tj|_||_tjd|_ |p$tj |j d|_ i|_ i|_i|_i|_i|_d|_d|_d|_|jj|_d|_d|_i|_i|_i|_i|_i|_g|_i|_| dS)z Initialize the dirty arbiter. Args: cfg: Gunicorn config log: Logger socket_path: Path to the arbiter's Unix socket pidfile: Well-known PID file location for orphan detection Nzgunicorn-dirty-)prefixz arbiter.sockrT)!cfglogpidosgetpidppidpidfiletempfilemkdtemptmpdirpathjoin socket_pathworkersworker_socketsworker_connections worker_queuesworker_consumers_worker_rr_index worker_agealive dirty_workers num_workers_server_loop_pending_requests app_specsapp_worker_mapworker_app_map_app_rr_indices_pending_respawns stash_tables_parse_app_specs)selfr(r)r4r.s r"__init__zDirtyArbiter.__init__BsIKK  &.?@@@ & "',, K+ +   "$ " ! 81  !#  !!# r$c|jjD]~}t|\}}|D t|}n3#t$r&}|jd||Yd}~nd}~wwxYw|||d|j|<t|j |<dS)a Parse all app specifications from config. Populates self.app_specs with parsed information about each app, including the import path and worker count limits. Worker count priority: 1. Config override (e.g., "module:Class:2") - highest priority 2. Class attribute (e.g., workers = 2 on the class) 3. None (all workers) - default Nz,Could not read workers attribute from %s: %s) import_path worker_count original_spec) r( dirty_appsrr Exceptionr)warningrBsetrC)rIspecrLrMes r"rHzDirtyArbiter._parse_app_specsysH' 5 5D(> r$c&g}|jD]t\}}|d}t|j|t }|||Y||kr||u|S)a Determine which apps a new worker should load. Returns a list of import paths for apps that need more workers. Apps with workers=None (all workers) are always included. Apps with worker limits are included only if they haven't reached their limit yet. Returns: List of import paths to load, or empty list if no apps need workers rM)rBitemslenrCgetrRappend)rI app_pathsrLrSrMcurrent_workerss r"_get_apps_for_new_workerz%DirtyArbiter._get_apps_for_new_workers !%!5!5!7!7 . . K/L!$"5"9"9+suu"M"MNNO#  ---- <//  ---r$ct||j|<|D]A}||jvrt|j|<|j||BdS)a? Register which apps a worker has loaded. Updates both app_worker_map and worker_app_map to track the bidirectional relationship between workers and apps. Args: worker_pid: The PID of the worker app_paths: List of app import paths loaded by this worker N)listrDrCrRaddrI worker_pidr_app_paths r"_register_worker_appsz"DirtyArbiter._register_worker_appssn+/y//J'! : :Ht22203#H-   ) - -j 9 9 9 9 : :r$c|j|g}|D]+}||jvr |j||,dS)z Unregister a worker's apps when it exits. Removes the worker from all tracking maps. Args: worker_pid: The PID of the worker to unregister N)rDpoprCdiscardres r"_unregister_workerzDirtyArbiter._unregister_workersf'++J;; " B BH4...#H-55jAAA B Br$c tj|_|jd|j|jr t |jd5}|t|jdddn #1swxYwYn2#t$r%}|j d|Yd}~nd}~wwxYw|j tj d<|j ||t!jd t%j|n#t*$rYnwxYw|dS#|wxYw)z&Run the dirty arbiter (blocking call).z Dirty arbiter starting (pid: %s)wNzFailed to write PID file: %sGUNICORN_DIRTY_SOCKETz dirty-arbiter)r+r,r*r)infor.openwritestrIOErrorrQr4environr(on_dirty_starting init_signalsr _setproctitleasynciorun _run_asyncKeyboardInterrupt _cleanup_sync)rIfrTs r"rzzDirtyArbiter.runs9;;  8$(CCC < D D$,,,+GGCMM***+++++++++++++++ D D D  !?CCCCCCCC D/3.> *+ ""4(((  ?+++ ! K)) * * * *     D     D   sfB(B > B BBBB C CC&EE, EE,EE,,Fc|jD]!}tj|tj"tjtj|jtjtj|jtjtj|jtjtj|jtjtj|jtjtj |jtjtj |jtjtj |jdS)zSet up signal handlers.N) SIGNALSrSIG_DFLSIGTERM_signal_handlerSIGQUITSIGINTSIGHUPSIGUSR1SIGCHLDSIGTTINSIGTTOU)rIsigs r"rwzDirtyArbiter.init_signals s< / /C M#v~ . . . . fnd&:;;; fnd&:;;; fmT%9::: fmT%9::: fnd&:;;; fnd&:;;; fnd&:;;; fnd&:;;;;;r$c|tjkr&jrjfddS|tjkrjdS|tjkr&jrjfddS|tjkrVxj dz c_ j dj jrjfddS|tj kr }j |krj d|dSxj dzc_ j dj jrjfd dSd _jr!jjdSdS) zHandle signals.cPtjSN)ry create_task_handle_sigchldrIsr"z.DirtyArbiter._signal_handler..sG/0D0D0F0FGGr$NcPtjSr)ryrreloadrsr"rz.DirtyArbiter._signal_handler..+sG/ >>r$rz'SIGTTIN: Increasing dirty workers to %scPtjSrryrmanage_workersrsr"rz.DirtyArbiter._signal_handler..6G/0C0C0E0EFFr$zASIGTTOU: Cannot decrease below %s workers (required by app specs)z'SIGTTOU: Decreasing dirty workers to %scPtjSrrrsr"rz.DirtyArbiter._signal_handler..Irr$F)rrr@call_soon_threadsaferr) reopen_filesrrr>rprrYrQr< _shutdown)rIrframe min_workerss` r"rzDirtyArbiter._signal_handlers& &. z  //GGGG F &. H ! ! # # # F &-  z  //>>>> F &.    !   HMMC* , , ,z  //FFFF F &. 3355K;..  .     !   HMMC* , , ,z  //FFFF F : < J + +DN ; ; ; ; ; < >$* + + ( Id& ' ' '%6  !          !5)))  5t7GHHH!!#########*4+?+?+A+ABB  | 3 3 3 3 3 3 3 3l00222222222 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3& 5    D     ! ! ! """"""""")    ))++             ! ! ! """"""""")    ))++        s> E D=+E= EE E EF:E(%F:'E((F:F FF:H GH G+(H *G++H clK|jrtjdd{Vtj|jkr7|jdd|_|dS| d{V| d{V|jdSdS)z1Periodically check worker health and manage pool.g?Nz+Parent changed, shutting down dirty arbiterF) r<rysleepr+getppidr-r)rQrmurder_workersrrs r"rzDirtyArbiter._worker_monitorsj (-$$ $ $ $ $ $ $ $z||ty((  !NOOO"    %%'' ' ' ' ' ' ' '%%'' ' ' ' ' ' ' 'j ( ( ( ( (r$cxK||jr|d{VdSdS)z#Handle SIGCHLD - reap dead workers.N) reap_workersr<rrs r"rzDirtyArbiter._handle_sigchldsV  : (%%'' ' ' ' ' ' ' ' ' ' ( (r$cK|jd |jr tj|d{V}n#t j$rYnwxYw|d}|tjkr| ||d{Vnv|tj kr| ||d{VnI|tj kr| ||d{Vn|||d{V|jn2#t$r%}|jd|Yd}~nd}~wwxYw| |d{VdS#t$rYdSwxYw#| |d{Vw#t$rYwwxYwxYw)a  Handle a connection from an HTTP worker. Routes requests to available dirty workers and returns responses. Supports both regular responses and streaming (chunk-based) responses. Also handles stash (shared state) operations. z&New client connection from HTTP workerNtypezClient connection error: %s)r)debugr<r read_message_asyncryIncompleteReadErrorr]MSG_TYPE_STASHhandle_stash_requestMSG_TYPE_STATUShandle_status_requestMSG_TYPE_MANAGEhandle_manage_request route_requestrPerrorr wait_closed)rIreaderwritermessagemsg_typerTs r"rzDirtyArbiter.handle_clientsu ?@@@ * >$1$DV$L$LLLLLLLGG2E#;;v..};;;33GVDDDDDDDDDD!>>>44WfEEEEEEEEEE!>>>44WfEEEEEEEEEE,,Wf========='* >( = = = HNN8! < < < < < < < < = LLNNN ((***********     LLNNN ((**********    sDADADACDF  E E;F EF E99 FF G  F;:G ; GG GG cK|dd}|d}||d{V}|l|jstd}n(|r|jrt |}ntd}t ||}tj||d{VdS||j vr| |d{V|j |}tj } |||| fd{V | d{VdS#t$rH} t |t!d| |}tj||d{VYd} ~ dSd} ~ wwxYw)a Route a request to an available dirty worker via queue. Each worker has a dedicated queue and consumer task. Requests are submitted to the queue and processed sequentially by the consumer. For streaming responses, messages (chunks) are forwarded directly to the client_writer as they arrive from the worker. Args: request: Request message dict client_writer: StreamWriter to send responses to client idunknownrgNzNo dirty workers availablezRequest failed:  worker_id)r]_get_available_workerr5rrBrr r write_message_asyncr8_start_worker_consumerryr create_futureputrPr ) rIrequest client_writer request_idrgrfrresponsequeuefuturerTs r"rzDirtyArbiter.route_requests [[y11 ;;z** 55h????????  < A"#?@@ Adn A4X>>"#?@@*:u==H3M8LL L L L L L L L F T/ / /--j99 9 9 9 9 9 9 9":.)++99;;ii-8999999999 MLLLLLLLLL M M M* !7A!7!7:NNNH 3M8LL L L L L L L L L L L L L L  Ms,D66 F=FFcKtjj<fd}tj|}|j<dS)z3Start a consumer task for a worker's request queue.cKjr d{V\}}} ||d{V|s|dn@#t $r3}|s||Yd}~nd}~wwxYwn#wxYwn#tj $rYdSwxYwjdSdSr) r<r]_execute_on_workerdone set_resultrP set_exception task_doneryr)rrrrTrrIrfs r"consumerz5DirtyArbiter._start_worker_consumer..consumers`* ;@99;;5F5F5F5F5F5F2G]F *"55& &{{}}4"--d333$444%{{}}4"003334)))))))))-EE*     sMC!AA21C2 B/<)B*%C*B//C2C!CC!!C43C4N)ryQueuer8rr9)rIrfrtaskrs`` @r"rz#DirtyArbiter._start_worker_consumerso ).:&       $"88::..,0j)))r$c^K|dd} ||d{V\}}tj||d{V t jtj||jjd{V}nY#tj $rGt|td|jj}tj||d{VYdSwxYw|d} | tj krtj||d{V| tj krtj||d{VdS| tjtjfvrtj||d{VdS|jd| U#t$$ry} |jd || ||t|t+d | | }tj||d{VYd} ~ dSd} ~ wwxYw) a Execute request on a specific worker (called by consumer). Handles both regular responses and streaming (chunk-based) responses. For streaming, chunk and end messages are forwarded directly to the client_writer as they arrive from the worker. rrNT)timeoutzWorker timeoutrz$Unknown message type from worker: %sz Error executing on worker %s: %szWorker communication failed: r)r]_get_worker_connectionr rrywait_forrr( dirty_timeout TimeoutErrorr r MSG_TYPE_CHUNK MSG_TYPE_ENDMSG_TYPE_RESPONSEMSG_TYPE_ERRORr)rQrPr_close_worker_connectionr ) rIrfrrrrrrrrrTs r"rzDirtyArbiter._execute_on_worker s[[y11 0 M#'#>#>z#J#JJJJJJJNFF3FGDD D D D D D D D! S $+$4%8@@ $ 6%%%GG+2")*:DH>>';M7SSSSSSSSSF  !GRRRC! SF M M M HNN=z1 M M M  ) )* 5 5 5* !D!D!D+5777H  3M8LL L L L L L L L L L L L L L MsJ:F)8B F)AC$ F)#C$$A/F)5F) F)) H,3A.H''H,cK|r-|jr&||jvrt|j|}n(dSt|j}|sdS|r@|jr9|j|d}|dzt|z|j|<n!|j}|dzt|z|_||t|zS)a Get an available worker PID using round-robin selection. If app_path is provided, only returns workers that have loaded that specific app. Uses per-app round-robin to ensure fair distribution among eligible workers. Args: app_path: Optional import path of the target app. If None, returns any worker using global round-robin. Returns: Worker PID or None if no eligible workers are available. Nrr) rBrCrcr5keysrEr]r\r:)rIrg eligible_pidsidxs r"rz"DirtyArbiter._get_available_workerGs  6 64... $T%8%B C C t!!2!2!4!455M 4  C C&**8Q77C.1Ag]9K9K-KD  * *'C%(1WM0B0B$BD !S3}#5#5566r$cK||jvr |j|S|j|}|std|t dD]=}t j|rn-tj dd{V>td|tj |d{V\}}||f|j|<||fS)z%Get or create connection to a worker.zNo socket for worker 2皙?NzWorker socket not ready: ) r7r6r]rranger+r2rryropen_unix_connection)rIrfr4_rrs r"rz#DirtyArbiter._get_worker_connectionqs  0 0 0*:6 6)--j99  CAZAABB Br H HAw~~k** -$$ $ $ $ $ $ $ $ $FFFGG G&;KHHHHHHHH/5v.> +v~r$c~||jvr3|j|\}}|dSdS)zClose connection to a worker.N)r7rjr)rIrf_readerrs r"rz%DirtyArbiter._close_worker_connectionsC 0 0 0"599*EEOGV LLNNNNN 1 0r$c K|dd}tj}g}|jD]\}} |j}t||z d} n#tttf$rd} YnwxYw| ||j t|dgt|dd| d|d |j|t!||jr&t%|jngd } t)|| } t+j|| d{VdS) z Handle a status query request. Returns information about the dirty arbiter and its workers. Args: message: Status request message client_writer: StreamWriter to send response to client rrNr_bootedF)r*ageappsrlast_heartbeatc|dS)Nr)rns r"rz4DirtyArbiter.handle_status_request..s %r$key) arbiter_pidr5rMr)r]time monotonicr5r[tmp last_updateroundOSError ValueErrorAttributeErrorr^rrsortr*r\rBrcrr r r) rIrrrnow workers_infor*workerrrresultrs r"rz"DirtyArbiter.handle_status_requests[[y11 n <--//  KC &$j4466 !&s['8!!ID,,..///r   !V44/ xHHHHHHHHHHHs ,A::BBc vK|dd}|d}tdt|dd} |tkrd}t |D]G}}|xjdz c_|dz }tjdd{VH|dkr#d d |dd tj jd }nVd d ||tj jd }n4|tkr } d} t |D]}j| krntj dkrn{xjdzc_tj fd} | t j| dz } tjdd{Vd d|| tj jd}n?t%d|} t'|| } t)j|| d{VdSjd|tkrd nd||d|ddt1||} t)j|| d{VdS#t2$rk}jd|t'|t%t7|} t)j|| d{VYd}~dSd}~wwxYw)z Handle a worker management request. Supports adding or removing dirty workers via protocol messages. Args: message: Manage request message client_writer: StreamWriter to send response to client rroprcountrNrTrdz)All apps have reached their worker limits)success operation requestedspawnedreason total_workerstarget_workers)rrrrrrc(j|jSrr5rprIs r"rz4DirtyArbiter.handle_manage_request..s4<?3Fr$rremove)rrrremovedrrzUnknown manage operation: z6Worker management: %s %d workers (spawned/removed: %d)rrzManage operation error: %s)r]rWintrr spawn_workerr>ryrr\r5rrYminr kill_workerrrrr r rr)rpr rPrrs)rIrrrrrrrrrr oldest_pidrrrTs` r"rz"DirtyArbiter.handle_manage_requests[[y11 [[  As7;;w223344N M]""u--A!..00F)((A-((1 !-,,,,,,,,,,a<<#'%*%*#$"M),T\):):*.*:FF$(%*%*#*),T\):):*.*: FF'''"7799 u - -A';664<((A--$$)$$"%T\%6%6%8%8)F)F)F)F"H"H"HJ$$Z@@@qLG!-,,,,,,,,,, $!)!&&%(%6%6&*&6 ##D#D#DEE.z5AA#7 xPPPPPPPPP HMMR#%#6#6%%H **Y 9a0H0HII K K K %Z88H3M8LL L L L L L L L L L M M M HNN7 ; ; ;*:z#a&&7I7IJJH3M8LL L L L L L L L L L L L L L Ms!!G!KA=K L8 A L33L8c dK|dd}|d}|dd}|d}|d}|d d }|tkr'||jvr i|j|<||j||<d }n#|tkr9||jvrd d i}n ||j|vrd d i}n|j||}n|tkr.||jvr!||j|vr|j||=d }nd }n|t krM||jvrg}nt |j|} rfd| D} | }nN|tkr,||jvr|j| d }n|tkr,||jvrd di}nt|j||d}n|tkr||jvr i|j|<d }n|tkr||jvr |j|=d }nd }n|tkr't |j}nk|tkr!||jvrd }nT|d }nO||j|v}n?t!d|} t#|| } t%j|| d {Vd St)|t*rd |vr|d } | dkrt!d|} n5| d krt!d|} nt!t-|} d| ddd| _t#|| } nt5||} t%j|| d {Vd S#t6$rk} |jd| t#|t!t-| } t%j|| d {VYd } ~ d Sd } ~ wwxYw)a! Handle a stash operation directly in the arbiter. All stash tables are stored in arbiter memory for simplicity and fast access. Args: message: Stash operation message client_writer: StreamWriter to send response to client rrrtablervaluepatternNTr key_not_foundFcXg|]&}tjt|$|'Sr)fnmatchrs)r kr)s r"r#z5DirtyArbiter.handle_stash_request..HsF$I$I$I!'.s1vvw'G'G$IA$I$I$Ir$table_not_found)sizer&zUnknown stash operation: zTable not found: zKey not found: StashrErrorzStash operation error: %s)r]rrGrrrrcrrclearrr\rrrrrr r r isinstancedictrstitlereplace error_typer rPr)r)rIrrrrr&rr(rall_keysrrr7rTr)s @r"rz!DirtyArbiter.handle_stash_requests [[y11 [[   GR((kk%   G$$++i((a MF\!! 111/1D%e,05!%(-|## 111%7FF 1% 888%7FF!.u5c:FF&&D---#9J59Q2Q2Q)%05!FF"FF}$$ 111FF#D$5e$<$A$A$C$CDDHI$I$I$I$Ix$I$I$I%FF~%%D---%e,22444}$$ 111%'89FF!$D$5e$< = =!&FF && 111/1D%e,,,,D---)%0!FF"FF&&d/446677&& 111"FF[!FF D$5e$<'>'>??EE&s6{{33E#U:+;+;+=+=+E+Ec2+N+N#U#U#U .z5AA(V<<3M8LL L L L L L L L L L M M M HNN6 : : :*:z#a&&7I7IJJH3M8LL L L L L L L L L L L L L L Ms!IN:CN:: P/A P**P/c8KjsdSj}jrhtj|krP}|n9t jdd{Vjrtj|kPtj|krtjfd} |tj t jdd{Vtj|k}dSdS)z%Maintain the number of dirty workers.Nrc(j|jSrrrs r"rz-DirtyArbiter.manage_workers..s4<?+>r$r) r<r>r\r5r!ryrr"rr#rr)rIr>rr$s` r"rzDirtyArbiter.manage_workerss@z  F& j %S..<<&&((F~-$$ $ $ $ $ $ $ $ j %S..<<$,+--T\..00!>!>!>!>@@@J   Z 8 8 8-$$ $ $ $ $ $ $ $ $,+------r$Fc|jr|jd}n=|r't|j}n|}|s|jddS|xjdz c_tj |j d|jd}t|j|j||j|j|}tj}|dkrj||_||j|<||j|<||||j|||jd|||Stj|_ t/jd |jjd |tjddS#t8$r-}tj|j|jndYd}~dSd}~wt<$rR|jd |j stj|j!tjdYdSwxYw) a Spawn a new dirty worker. Worker app assignment follows these priorities: 1. If there are pending respawns (from dead workers), use those apps 2. Otherwise, determine apps for a new worker based on allocation 3. If force_all_apps=True, spawn with all apps regardless of limits Args: force_all_apps: If True, spawn worker with all apps ignoring limits Returns: Worker PID in parent process, or None if no apps need workers rz)No apps need more workers, skipping spawnNrzworker-z.sock)rr-r_r(r)r4z,Spawned dirty worker (pid: %s) with apps: %szdirty-worker []z!Exception in dirty worker process)"rFrjrcrBrrar)rr;r+r2r3r1rr*r(forkr5r6rhdirty_post_forkrpr,rrx proc_name init_process_exit SystemExitcoderP exceptionrWORKER_BOOT_ERROR)rIforce_all_appsr_r4rr*rTs r"r!zDirtyArbiter.spawn_workersv  ! 8.22155II  8T^002233II5577I  HNNF G G G4 1gll K94?999  #    gii !88FJ &DL '2D  $  & &sI 6 6 6 H $ $T6 2 2 2 HMMHy * * *JY[[    E0BEEE F F F    ! ! ! HQKKKKK : : : Hqv1QVVq 9 9 9 9 9 9 9 9 9    H  B C C C= 1/000 HQKKKKKK  s A G I"G??AIIc tj||dS#t$r;}|jtjkr||Yd}~dSYd}~dSd}~wwxYw)zKill a worker by PID.N)r+killrerrnoESRCH_cleanup_worker)rIr*rrTs r"r#zDirtyArbiter.kill_workers * GC      * * *w%+%%$$S)))))))))&%%%%% *s A*AAc||||jvr'|j||j|=|j|d||jvr6t |j|}|r|j|| ||j |d}|r|j |||j |d}|rGtj|r* tj|dS#t$$rYdSwxYwdSdS)z Clean up after a worker exits. Saves the dead worker's app list to pending respawns so the replacement worker gets the same apps. N)rr9rr8rjrDrcrFr^rlr5r(dirty_worker_exitr6r+r2rrr)rIr* dead_appsrr4s r"rKzDirtyArbiter._cleanup_workers %%c*** $' ' '  !# & - - / / /%c* sD))) $% % %T0566I 9&--i888 $$$!!#t,,  5 H & &tV 4 4 4)--c488  27>>+66   +&&&&&        s*E E EcK|jjsdSt|jD]\}} t j|jz |jjkrBn#ttf$rYVwxYw|j sC|j d|d|_ ||tj||tjdS)z!Kill workers that have timed out.NzDIRTY WORKER TIMEOUT (pid:%s)T)r(rrcr5r[rrrrrr abortedr)criticalr#rSIGABRTSIGKILL)rIr*rs r"rzDirtyArbiter.murder_workerss x%  F 2 2 4 455 6 6KC >##fj&<&<&>&>>$(BXXXYZ(    > 6!!"A3GGG!%  fn5555  fn5555 6 6s;A::B Bc: tjdtj\}}|sdSd}tj|rtj|}nDtj|r0tj|}|jd||||j kr|j d|| ||j d|#t$r!}|jtjkrYd}~dSd}~wwxYw)zReap dead worker processes.TNz)Dirty worker (pid:%s) killed by signal %sz$Dirty worker failed to boot (pid:%s)zDirty worker exited (pid:%s))r+waitpidWNOHANG WIFEXITED WEXITSTATUS WIFSIGNALEDWTERMSIGr)rQrErrKrprrIECHILD)rIwpidstatusexitcoderrTs r"rzDirtyArbiter.reap_workers*s;  D!z"bj99 fE<''0!~f55HH^F++0+f--CH$$%P%)3000t555HNN#I4PPP$$T*** >$* + +   $*++++     Z ,, 8 8 "',,t{A667777 HT[ ! ! ! !    D   7BBBBBs6A AA<B B#"B#'A,D D! D!)NNr)F)T)'__name__ __module__ __qualname____doc__splitrrErJrHrYrarhrlrzrwrrr{rrrrrrrrrrrrrr!r#rKrrrrr}rr$r"rr1sxEE;AACCEEEG5 5 5 5 n 5 5 5D"6:::$BBB"!!!@ < < <8<8<8rrs    @@@@@@@@ " OCOCOCOCOCOCOCOCOCOCr$