
    i                       d dl mZ ddlmZ ddlmZ  e                                 e	e                      e                                 e	e                     dadbdZ
dcdddZdedZ	 dadfdZdgdZdhd!Zdid$Zdjd(Zefdkd-Zdld/Zdmd2Zdnd3Zdod4Zdod5Zdpd8Zefdqd;Zdrd<Zefdsd@ZefdkdAZdtdBZdudCZ dvdEZ!dwdIZ"dxdKZ#dydNZ$dzdOZ%	 d{d|dVZ&dzdWZ'd}dYZ(dad~d[Z)dd`Z*dS )    )annotations   )	bootstrap)	editorialNpromptstrapi_key
max_tokens
int | Nonereturnc                	   t          t                              d          pt                              d          pd                                          pd}|d| dgddd}d }t          t                              d	          pd
                                          }|r#	 t	          |          }n# t
          $ r d }Y nw xY w||dk    rt	          |          |d<   d| dd}d}t          t                              d          pt                              d          pd
                                          }	|	r#	 t          |	          }n# t
          $ r d}Y nw xY wt          dt          d|                    }t	          t          t                              d          pd          pd          }
t          t                              dd          pd                                          	                                dv }d}	 |dz  }	 t                              t          |||          }|                                 |                                }|                    d          }t!          |t"                    r|rt!          |d         t$                    r|d         ni }t!          |                    d          t$                    r|                    d          ni }|                    d          }t!          |t                     r(|                                r|                                S |                    d           p)|                    d!          p|                    d"          }t!          |t                     r(|                                r|                                S t!          |                    d#          t$                    r|                    d#          ni }t          |                    d$          pd
                                          }|rt'          d%|           t'          d&|           # t
          $ r}t          |pd
          }|	                                }d'|v pd(|v pd)|v pd*|v pd+|v }|r |s|
dk    r||
k    r t          d,d-d.t          d|dz
            z  z            }t(                              |           Y d }~nd }~ww xY w)/NMINIMAX_TEXT_MODELMINIMAX_MODELzMiniMax-M2.7user)rolecontentg333333?F)modelmessagestemperaturereasoning_splitMINIMAX_TEXT_MAX_TOKENS r   r
   zBearer zapplication/json)AuthorizationzContent-Typeg     A@MINIMAX_HTTP_TIMEOUT_SECONDSMINIMAX_TIMEOUT_SECONDSg       @g      ^@MINIMAX_RETRY_MAX_ATTEMPTS0MINIMAX_RETRY_FOREVER1>   r   onyestrueTr   )headersjsontimeoutchoicesmessager   replyoutput_texttext	base_resp
status_msgzMiniMax response error: z(MiniMax response missing content; model=z
login failauthorizationzapi secret key401403g      4@g      ?g?)r   osgetenvstripint	ExceptionfloatmaxminlowerrequestspostMINIMAX_URLraise_for_statusr$   get
isinstancelistdictRuntimeErrortimesleep)r   r	   r
   
model_namepayloadtoken_limit	env_limitr#   timeout_secenv_timeoutmax_attemptsretry_foreverattemptrespdatar&   firstr'   r   directr+   r,   errormsg	msg_lower
auth_error	sleep_secs                              /app/scripts/x_intel/sources.pyminimax_chatrW      s:   
		&'' 	99_%%	  egg	   	  $889 	 G KBII788>B??EEGGI 	i..KK 	 	 	KKK	;?? #K 0 0,7,,* G K
		011 	99.//	  egg	 
  	,,KK 	 	 	KKK	c3uk2233Ks299%ABBIcJJOaPPL		"93??F3GGMMOOUUWW[uuMG$"1"	"==gGU`=aaD!!###99;;Dhhy))G'4(( +W +&0T&B&BJ

2<UYYy=Q=QSW2X2X`%))I...^`!++i00gs++ + +"==??*XXg&&U$((=*A*AUTXXfEUEUF&#&& &6<<>> &||~~%1;DHH[<Q<QSW1X1X`---^`IY]]<88>B??EEGGJ L"#Jj#J#JKKKV*VVWWW 	" 	" 	"ekr""C		I	) &"i/&#y0& I%& I%     \A%5%5'\:Q:QD#Aw{0C0C)C"DEEIJJy!!!!!!!!	"+$"sL   #B3 3CCE E E D,Q ?A;Q ;BQ 
S6 BS11S6r   cardslist[StoryCard]feedback_contextNonec           
     |
   | D ]7}d|rd| dndz   d|j          d|j         d|j        d d          z   }	 t          ||          }t	          |          }|sAd	|j          d
|j         d|j        d d          }t          ||          }t	          |          }|st          |                    d          pd                                          }t          |                    d          pd                                          }	t          |                    d          t                    r|                    d          ng }
t          |                    d          pd                                          
                                }t          |                    d          pd                                          
                                }t          |                    d          t                    r|                    d          ng }|                    d          }t          |                    d                    }t          |                    d                    }t          t          |                    d          pd                    d d         }t          |                    d          d          }|r|d d         |_        |	r|	d d         |_        |
r d |
D             d d         p|j        |_        |dv r||_        |d v r||_        |rd! |D             d d         |_        t          |t*          t,          f          r0t-          t/          d"t1          d#|                              |_        |j        d$k    r	|r||_        |r||_        |r||_        |r||_        t=          |j        |j                  d%k    sqt?          |j        |j                  sWtA          d& |j        pg D             |j                  s1|j        d'k    rtC          |j        d( |j        pg D                       rtE          |j        |j        |j                   }t          |                    d          p|j                  d d         |_        |                    d          }t          |t                    r|rd) |D             d d         |_        tG          |           tI          |          |_%        tM          |           tO          |d*+           (# tP          $ r Y 5w xY wd S ),Nu  你是TCG社群編輯。請先完整讀懂內容，再輸出『非抄寫』重整版本。輸出必須是 JSON，欄位固定為：title,summary,bullets(長度3),card_type(layout可選:poster/brief/data/timeline),confidence(0~1),tags(最多3),event_facts(可選，僅 event 使用: reward/participation/audience/location/schedule),topic_labels(可多選: events/official/sbt/pokemon/alpha/tools/other)。限制：1) 不可逐句複製原文；2) summary 要用第三人稱重述；3) bullets 每條都要是可行動或可追蹤的資訊；4) card_type 只能是 event/feature/announcement/market/report/insight；5) 必須用語意判斷分類，不可只用關鍵字；6) 只有含明確活動訊號（時間/地點/報名/參與方式）才可標為 event；7) 產品進度、版本更新、開放計畫優先標為 feature 或 announcement，不算 event；8) 單句互動、祝賀、表情、聊天回覆通常是 insight；9) topic_labels 可以多選，允許同時屬於 events 與 sbt（例如活動獎勵包含 SBT）；10) 繁體中文，不可捏造；11) 禁止使用『核心訊號/關鍵數字/決策建議/判讀建議/分析主題/文中數據/使用方式』這種模板詞；12) 若出現數字，必須說明它代表什麼（單位/情境/用途），不能只列數字；13) summary 需涵蓋『發生了什麼、為何重要、影響誰、下一步該看什麼』；14) 若為串文(thread)或多段內容，先整合後再輸出單一版本；15) 禁止空話（例如『社群互動貼文、重點在現場動態與回饋』），必須寫出實際更新內容；16) 禁止猜測語氣（例如『可能/通常/推測/大概』），除非原文明確使用該語氣；17) 若提到數字，必須同句交代該數字對應的對象與意義（例如價格、版本、名額、成交）；18) 不可使用 Markdown code fence（```）；19) 長度限制：title<=40字、summary<=150字、每條bullet<=34字；20) 若使用者回饋記憶與原始推斷衝突，以使用者回饋記憶優先；21) 整份 JSON 請控制在約 800 字元內。

u   [使用者回饋記憶]


r   u   來源帳號: @u   
來源URL: u	   
內容: ih  u;  請直接輸出合法 JSON，不要任何前後文字，不要 ```。欄位固定：title,summary,bullets(3),card_type,layout,tags,confidence,event_facts,topic_labels。全部繁體中文，且每欄位要短：title<=40字、summary<=120字、每條bullet<=30字。不可捏造，需依據提供內容。

帳號:@z
URL:u   
內容:i  titlesummarybullets	card_typelayouttags
confidenceevent_factstopic_labelsdetail_summaryi  detail_lines   limitx   i@  c                    g | ]G}t          |                                          #t          t          |                    d d         HS Nrl   r   r2   
clean_text.0xs     rV   
<listcomp>z.apply_minimax_story_refine.<locals>.<listcomp>   sC    \\\QSQRVV\\^^\
3q66 2 24C4 8\\\       >   eventmarketreportfeatureinsightannouncement>   rN   briefpostertimelinec                    g | ]G}t          |                                          #t          t          |                    d d         HS )N   ro   rq   s     rV   rt   z.apply_minimax_story_refine.<locals>.<listcomp>   sB    UUUc!ffllnnUZA//4UUUru           g      ?rw   gq=
ףp?c                F    g | ]}t          t          |                    S  rp   r   rq   s     rV   rt   z.apply_minimax_story_refine.<locals>.<listcomp>   s&    )[)[)[*SVV*<*<)[)[)[ru   rx   c                F    g | ]}t          t          |                    S r   r   rq   s     rV   rt   z.apply_minimax_story_refine.<locals>.<listcomp>   sD      YK  YK  YKpqYcdghidjdjYkYk  YK  YK  YKru   c                    g | ]G}t          |                                          #t          t          |                    d d         HS rn   ro   rq   s     rV   rt   z.apply_minimax_story_refine.<locals>.<listcomp>   sC    #[#[#[CPQFFLLNN#[Js1vv$6$6tt$<#[#[#[ru   T)preserve_type))accounturlraw_textrW   parse_json_blockr   r=   r2   r>   r?   r8   normalize_event_factsnormalize_topic_labelsrp   normalize_detail_linesr^   r_   r`   ra   rb   rc   r3   r5   r6   r7   rd   re   rf   rg   rh   similarity_ratio_summary_needs_rewrite_bullets_need_rewrite_market_is_number_dumpbuild_editorial_copyenrich_detail_view
score_card
importanceenrich_card_metadatanormalize_card_semanticsr4   )rX   r	   rZ   cardr   rawparsedcompact_retry_promptr^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   fallbackfbs                        rV   apply_minimax_story_refiner   Z   s    e eC6 FV]A+;AAAA[]7_8. . .(. .}UdU+. .9. 	BC	vw//C%c**F /5  $|	5 5
  85 5 #mETE25 5 % ##7AA)#.. 

7++1r2288::E&**Y//5266<<>>G/9&**Y:O:OQU/V/V^fjj+++\^GFJJ{339r::@@BBHHJJIH--344::<<BBDDF)3FJJv4F4F)M)MU6::f%%%SUDL11J/

=0I0IJJK1&**^2L2LMML'FJJ7G,H,H,NB(O(OPPQURUQUVN1&**^2L2LTUVVVL )"4C4[
 -&tt} q\\'\\\]_^_]_`pdhdp___!*@@@$ ZUUdUUUVXWXVXY	*sEl33 H"'CS*1E1E(F(F"G"G~(([(#.  1$0! 5&4# 1$0! t}==DD)$,FF E()[)[t|GYWY)[)[)[]a]jkk E Nh..3I$,  YK  YKvz  wC  wI  GI  YK  YK  YK  4L  4L./t~t|\\"8<<	#:#:#JdlKKDSDQ\\),,b$'' `B `#[#[b#[#[#[\^]^\^#_DLt$$$(..DO &&&$T>>>>> 	 	 	H	Ie es   A$T+RT++
T98T9r*   dict[str, Any] | Nonec                   | sd S |                                  } 	 t                              |           }t          |t                    r|nd S # t
          $ r Y nw xY wt                              d| t          j                  }|sd S 	 t                              |	                    d                    }t          |t                    r|nd S # t
          $ r Y d S w xY w)Nz\{.*\}r   )
r2   r$   loadsr>   r@   r4   researchSgroup)r*   r   matchs      rV   r   r      s     t::<<DD!!#FD11;vvt;   IIirt,,E tEKKNN++#FD11;vvt;   tts$   2A 
AAAC 
CCsectionsdict[str, list[dict[str, Any]]]	key_terms	list[str]
str | Nonedict[str, Any]c                   t          | d d          }d |d d         D             }ddg dt          d	 |D                       |d d
         d}|r|s|S dd                    |          z   dz   d                    d |                    dg           d d         D                       z   dz   d                    d |                    dg           d d         D                       z   dz   d                    d |                    dg           d d         D                       z   }	 t          ||          }t	          |          }	|	rt          |	                    d          p|d                   d d         |d<   t          |	                    d          p|d                   d d         |d<   |	                    d          }
t          |
t                    r|
rd |
D             d d         |d<   n# t          $ r Y nw xY w|S )Nc                    | j         S Npublished_atcs    rV   <lambda>z"aggregate_digest.<locals>.<lambda>   s    q~ ru   Tkeyreversec                2    g | ]}d |j          d|j         S )z- @: )r   r^   rr   r   s     rV   rt   z$aggregate_digest.<locals>.<listcomp>   s-    III1.	..QW..IIIru      u   Spring AI 關鍵情報總結u{   已從高訊號貼文中整理出官方更新、近期活動、即將開放與社群焦點，避免被零散回覆淹沒。)u3   先看官方更新，確認產品與活動方向。u<   近期活動用時間與參與方式呈現，減少漏看。u?   社群焦點只保留有訊息密度的貼文，不再全貼。c                    h | ]	}|j         
S r   )r   r   s     rV   	<setcomp>z#aggregate_digest.<locals>.<setcomp>   s    "C"C"C19"C"C"Cru      )headline
conclusion	takeawaysaccounts_activer   u   你是TCG情報總編。請根據貼文標題與四個情報分類，輸出 JSON：headline,conclusion,takeaways(長度3)。語氣要像『春季資訊刊』，但保持專業，繁體中文，不可捏造。


z

[official_updates]
c              3  ,   K   | ]}d |d          V  dS z- r   Nr   rq   s     rV   	<genexpr>z#aggregate_digest.<locals>.<genexpr>   s/      [[Q(:(([[[[[[ru   official_updates   z

[upcoming_events]
c              3  ,   K   | ]}d |d          V  dS r   r   rq   s     rV   r   z#aggregate_digest.<locals>.<genexpr>   s/      ZZQ(:((ZZZZZZru   upcoming_eventsz

[upcoming_features]
c              3  ,   K   | ]}d |d          V  dS r   r   rq   s     rV   r   z#aggregate_digest.<locals>.<genexpr>   s/      \\Q(:((\\\\\\ru   upcoming_featuresr   P   r      r   c                    g | ]G}t          |                                          #t          t          |                    d d         HS )NZ   ro   rq   s     rV   rt   z$aggregate_digest.<locals>.<listcomp>  sC    &^&^&^1sSTvv||~~&^z#a&&'9'9#2#'>&^&^&^ru   rv   )	sortedjoinr=   rW   r   r   r>   r?   r4   )rX   r   r   r	   cards_sorted
top_titlesdigestr   r   r   tkss              rV   aggregate_digestr      s    %%=%=tLLLLIIRaR8HIIIJ 3 T
 
 

 ""C"Cl"C"C"CDDss^
 
F  , 	c
))J

	  %	% ))[[(,,?QSU2V2VWYXYWY2Z[[[
[
[		\
 $	$ ))ZZ(,,?PRT2U2UVXWXVX2YZZZ
Z
Z	[ &	& ))\\(,,?RTV2W2WXZYZXZ2[\\\
\
\	] 
67++!#&& 	c!$VZZ
%;%;%Qvj?Q!R!RSVTVSV!WF:#&vzz,'?'?'W6,CW#X#XY]Z]Y]#^F< **[))C#t$$ c c&^&^&^&^&^_a`a_a&b{#    Ms   -CG< <
H	H	r   c                   t          d          sd S 	 t                              dd| dgt          j        t          j        ddd          }|j        dk    s|j                                        sd S t                              |j                  }t          |
                    d	          p|
                    d
          pd                                          }|sd S t          |
                    d          pd                                          }d| d|  d}|r	|d| dz  }|d| dz  }|S # t          $ r Y d S w xY w)Ntwittertweet--jsonT#   Fstdoutstderrr*   r%   checkr   	full_textr*   r   
created_atzTitle: X on X: "z" / X

URL Source: r   z
Published Time: z
Markdown Content:
)shutil_which
subprocessrunPIPE
returncoder   r2   r$   r   r   r=   r4   )r   procrN   r   createdr^   s         rV   fetch_status_with_twitter_clir     sr   	"" t~~h/??  
 
 ?at{'8'8':':4zz$+&&dhh{++Etxx/?/?E2FFLLNN 	4dhh|,,23399;;H7HHHHH 	65'5555E44444   tts    AD4 1A,D4 AD4 4
EEvaluedatetime | Nonec                ~   t          | pd                                          }|sd S |g}|                    d          r)|                    |                    dd                     |D ]t}	 t
                              |          }|j         |                    t          j	                  }|
                    t          j	                  c S # t          $ r Y qw xY wdD ]u}	 t
                              ||          }|j         |                    t          j	                  }|
                    t          j	                  c S # t          $ r Y rw xY w	 t          |          }|j         |                    t          j	                  }|
                    t          j	                  S # t          $ r Y d S w xY w)Nr   Zz+00:00tzinfo)z%a %b %d %H:%M:%S %z %Yz%a, %d %b %Y %H:%M:%S %Z)r   r2   endswithappendreplacedatetimefromisoformatr   timezoneutc
astimezoner4   strptimeparsedate_to_datetime)r   r   
candidatescanddtfmts         rV   parse_datetime_guessr  $  s   
ekr


 
 
"
"C tJ
||C 6#++c844555  	''--By ZZx|Z44==..... 	 	 	D	 G  	""3,,By ZZx|Z44==..... 	 	 	H	"3''98<00B}}X\***   tts9   .AC
CC%A E
EEAF. .
F<;F<r   list[dict[str, Any]]c                   | pd                                 }|sg S g }	 t                              |          }|                    |           n|# t          $ ro |                                D ]W}|                                 }|s	 |                    t                              |                     H# t          $ r Y Tw xY wY nw xY wg }|D ]'}t          |t                    r0|D ],}t          |t                    r|                    |           -Ht          |t                    r|	                    d          |	                    d          |	                    d          |	                    d          |	                    d          g}d}	|D ]H}
t          |
t                    r1d}	|
D ],}t          |t                    r|                    |           -I|	s|                    |           )|S )	Nr   tweetsitemsrN   resultsstatusesFT)
r2   r$   r   r   r4   
splitlinesr>   r?   r@   r=   )r   r*   parsed_objectsr   liner  objrs   candidate_listsexpandedarrs              rV   parse_twitter_cli_outputr  H  s1   Lb!!D 	 "ND!!f%%%%   OO%% 	 	D::<<D %%djj&6&67777   	 	 #%E " "c4   	 $ $a&& $LLOOOc4   	"!!  	""
##O H& , ,c4(( ,#H  , ,%a.. ,!LLOOO "S!!!Ls5   /A 7C-B43C4
C>C CCCitemusernameStoryCard | Nonec                 
   t          |                     d          p+|                     d          p|                     d          pd                                          }|sd S |                     d          pT|                     d          p?|                     d          p*|                     di                               d          pd}t          t          |                    }t	          |          d	k     st          |          rd S t          |                     d
          p@|                     d          p+|                     d          p|                     d          pd                                          }t          |          pt          |          }t          |                     d          pd                                          }|sd| d| }t          |                     d          t                    r|                     d          ni }t          |                    d|                     dd                    pd          t          |                    d|                     dd                    pd          t          |                    d|                     dd                    pd          t          |                    d|                     dd                    pd          t          |                    dd          pd          d}t          |                     d                    }	t          |                     d          p+|                     d          p|                     d           pd                                          }
t          |          \  }}}t          |||          }t          d3i d|d!|d|d"t          |                    d"          pt          |d#                    d$t          |                    d$          pt          |d%                    d&t          |                    d&          t                     r|                    d&          nt#          |          d'|                                d(d)d*|d+|d,|d-|d d.         d/d0d1|	d|d2|
}t'          |          |_        t+          |           t-          |           |S )4Nid_stridtweet_idr   r   r*   r   
note_tweetr   	createdAtr   daterB   r   https://x.com//status/metricslikesfavorite_countr   retweetsretweet_countrepliesreply_countquotesquote_countviewsr  r  r!  r#  r%  mediain_reply_to_status_id_strin_reply_to_status_idinReplyToStatusIdr   r^      r_     r`   r   rd   gffffff?ra   rb   rc   r   	  providertwitter-clicover_imagereply_to_idr   )r   r=   r2   rp   lenis_noise_textr  snowflake_to_datetimer>   r@   r3   extract_first_imageclassify_storyr   	StoryCardsummarize_naiver?   extract_bullets	isoformatr   r   r   r   )r  r  sidr*   created_raw
created_dtr   metrics_rawr  coverr1  ra   rb   rc   shapedr   s                   rV   %build_storycard_from_twitter_cli_itemrA  v  s   
dhhx  PDHHTNNPdhhz6J6JPb
Q
Q
W
W
Y
YC t 	 	88F	88I	 88L"%%))&11	  	 c$ii  D
4yy1}}d++}tdhh{++stxx/E/EsRXIYIYs]a]e]efl]m]msqsttzz||K%k22P6KC6P6PJ
dhhuoo#
$
$
*
*
,
,C 76x6666)3DHHY4G4G)N)NV$((9%%%TVK[__Wdhh7G.K.KLLQPQRR
DHH_a4P4PQQVUVWW{y$((=!2L2LMMRQRSSkooh0J0JKKPqQQ[__Wa005A66 G   1 122E,-- 	88+,,	88'((	 	 
 egg  -T22Ivt!$	8<<F   3 C &**W%%Cs)C)CDDD	
 FJJy))G_T3-G-GHHH *4FJJy4I4I4)P)Pk

9%%%VefjVkVk  ))+++ 3 ) v T ete  E    K!D$ !&&DOtKru   since_dtr   	max_postsr3   c           
     |   t          d          sg S t          |dz  d          }dd| ddt          |          gddd| ddt          |          d	d
g	g}g }t                      }|D ]6}	 t                              |t          j        t          j        ddd          }n# t          $ r Y Ew xY w|j        dk    s|j	        
                                snt          |j	                  D ]}	t          |	|           }
|
s	 t                              |
j                  |k     r;n# t          $ r Y nw xY w|
j        |v rV|                    |
j                   |                    |
           t'          |          |k    rt)          |d d          c c S |r n8|                    d d           |d |         S )Nr   r   $   z
user-postsr   z-nr   z--fromz	--excluder  T-   Fr   r   )r  c                    | j         S r   r   r   s    rV   r   z6fetch_account_cards_with_twitter_cli.<locals>.<lambda>  s    1> ru   r   c                    | j         S r   r   r   s    rV   r   z6fetch_account_cards_with_twitter_cli.<locals>.<lambda>      Q^ ru   )r   r6   r   setr   r   r   r4   r   r   r2   r  rA  r   r   r   r  addr   r2  r   sort)r  rB  rC  target_ncommandsrX   seen_idscmdr   r  r   s              rV   $fetch_account_cards_with_twitter_clirQ    s4   
 	"" 	9q="%%H	L(HdCMMJ	Hh(D#h--Q\^hiH
  EH ! !
	>>!! "  DD  	 	 	H	 ?at{'8'8':':,T[99 	Q 	QD8QQQD ))$*;<<xGG H   w(""LL!!!LL5zzY&&e)A)A4PPPPPPPP '  	E	 
JJ++TJ:::)s$   (4B
B*)B*=#D""
D/.D/rP  c                $    ddl m}  ||           S )Nr   )which)shutilrS  )rP  rS  s     rV   r   r     s"    5::ru   r  -tuple[str | None, str, dict[str, Any] | None]c                   d|  d| }t          |          }t          |          }|r|d|fS t          |t                    rt	          |                    d          pd                                                                                              d          }t	          | pd                                                                                              d          }||k    rIt	          |                    d          pd                                          rt          ||          d|fS 	 t          d	|  d|           d
|fS # t          $ r d d|fcY S w xY w)Nr  r  r/  r   r   @r*   ztweet-resultzhttps://r.jina.ai/http://x.com/z	r.jina.ainone)fetch_status_metadatar   r>   r@   r   r=   r2   r8   lstripbuild_markdown_from_status_meta
fetch_textr4   )r  r  r   metatwitter_cli_dataownerwanteds          rV   fetch_status_markdownra    sy   
78
7
7X
7
7C **D4S99 544$ TDHHY''-2..4466<<>>EEcJJX^$$**,,2244;;C@@F??s488F#3#3#9r::@@BB?24==~tSS"XHXXhXXYY[fhlll " " "VT!!!!"s   6E E E c                    t          t                              d          pt                              d          pd                                          } t          t                              d          pt                              d          pd                                          }g }t                              d|          D ]I}|                                }|rt                              d|          s4|                    |           Jt          t          
                    |                    }t          t                              d          pt                                                    }	 t          |          }n# t          $ r
 t          }Y nw xY wt          d	t          d
|                    }t          t                              d          pd                                                                          }|dv rd}n|dv rd}nt#          | o|          }t#          | o|          }	t#          |o|	          |	| ||dS )NDISCORD_BOT_TOKENDISCORD_TOKENr   DISCORD_MONITOR_CHANNEL_IDSDISCORD_MONITOR_CHANNEL_IDz[,\s]+z\d{6,}DISCORD_MONITOR_LIMIT
   d   DISCORD_MONITOR_ENABLED>   r   noofffalseF>   r   r    r!   r"   T)enabled
configuredtokenchannel_idsrk   )r   r0   r1   r2   r   split	fullmatchr   r?   r@   fromkeysDEFAULT_DISCORD_MONITOR_LIMITr3   r4   r6   r7   r8   bool)
rp  raw_channelschannelspiececid	raw_limitrk   enabled_rawrn  ro  s
             rV   resolve_discord_monitor_configr}    s-   		-..R"))O2L2LRPRSSYY[[E
		/00 	99122	  egg	 
 H)\22  kkmm 	",,y#66 	DMM(++,,HBII566W:WXX^^``I.I . . .-.CUOO$$Ebii 9::@bAAGGIIOOQQK111	2	2	2u)**e())J.J//   s   :F
 
FFc                   t          |                     d          pd          g}t          |                     d          t                    r|                     d          ng }|D ]i}t          |t                    sdD ]O}t          |                    |          pd                                          }|r|                    |           Pt          |                    d          t                    r|                    d          ng }|D ]}t          |t                    st          |                    d          pd                                          }t          |                    d          pd                                          }	d                    d	 ||	fD                       }
|
r|                    |
           kt          d                    |                    S )
Nr   r   embeds)r^   descriptionfieldsnamer    c              3     K   | ]}||V  	d S r   r   rq   s     rV   r   z(_discord_message_text.<locals>.<genexpr>?  s'      <<A!<a<<<<<<ru   )	r   r=   r>   r?   r@   r2   r   r   rp   )r  partsr  embedr   valr  fieldr  r   mergeds              rV   _discord_message_textr  /  s   DHHY//52667E#-dhhx.@.@$#G#GOTXXhRF % %%&& 	+ 	" 	"Ceiinn*++1133C "S!!!(2599X3F3F(M(MU8$$$SU 	% 	%EeT** uyy((.B//5577D		'**0b117799EXX<<$<<<<<F %V$$$	% chhuoo&&&ru   c                   t          |                     d          t                    r|                     d          ng }|D ]}t          |t                    st	          |                    d          pd                                          }|                    d          p!t          |                    d                    }|st	          |                    d          p|                    d          pd                                          }|                    d          r|c S t          |                     d	          t                    r|                     d	          ng }|D ]}t          |t                    st          |                    d
          t                    r|                    d
          ni }t          |                    d          t                    r|                    d          ni }	||	fD ]h}
t	          |
                    d          p|
                    d          pd                                          }|                    d          r|c c S idS )Nattachmentscontent_typer   zimage/width	proxy_urlr   httpr  image	thumbnail)	r>   r=   r?   r@   r   r8   
startswithrv  r2   )r  r  attr  is_imager   r  r  r  r  sources              rV   _discord_first_imager  E  sP   -78O8OQU-V-V^$((=)))\^K 	 	#t$$ 	377>228b99??AA**844NSWWW=M=M8N8N 	#''+&&>#''%..>B??EEGG>>&!! 	JJJ	#-dhhx.@.@$#G#GOTXXhRF  %&& 	&071C1CT&J&JR		'"""PR.8;9O9OQU.V.V^EIIk***\^	i( 	 	Ffjj--HE1B1BHbIIOOQQC~~f%% 




	 2ru   
channel_id
message_idc                    t          |                     d          pd                                          }|rd| d| d| S d| d| S )Nguild_idr   zhttps://discord.com/channels//z!https://discord.com/channels/@me/)r   r=   r2   )r  r  r  r  s       rV   _discord_message_urlr  ^  si    488J''-2..4466H TSxSS*SSzSSSHzHHJHHHru   rp  rk   c                   d| dd}dt          dt          |d                    i}t           d|  d}t                              |||d	
          }|j        dk    rJt          |j        pd          d d         }t          d|j         d| 	                                          |
                                }t          |t                    sg S d |D             S )NzBot zRenaissIntelDiscordMonitor/1.0)r   z
User-Agentrk   r   ri  z
/channels/z	/messages   )r#   paramsr%   i  r   rl   zHTTP r  c                <    g | ]}t          |t                    |S r   )r>   r@   rq   s     rV   rt   z2fetch_discord_channel_messages.<locals>.<listcomp>s  s'    333!z!T223A333ru   )r6   r7   DISCORD_API_BASE_URLr9   r=   status_coderp   r*   rA   r2   r$   r>   r?   )	r  rp  rk   r#   r  r   rM   bodyrN   s	            rV   fetch_discord_channel_messagesr  e  s    '6 G s1c%oo../F!
B
BZ
B
B
BC<<WVR<HHD3$)/r**4C40<4#3<<d<<BBDDEEE99;;DdD!! 	33t3333ru   c                   t          |                     d          pd                                          }|sd S t          |           }t	          |          dk     st          |          rd S t          |                     d          p|                     d          pd                                          }t          |          p#t                              t          j
                  }t          |                     d          t                    r|                     d          ni }t          |                    d          p+|                    d          p|                    d          pd	                                          }d}t          |                     d
          t                    r|                     d
          ni }	|	r6t          |	                    d          pd                                          }|sut          |                     d          t                    r|                     d          ni }
t          |
                    d          pd                                          }t          |          \  }}}t          |||          }t          d#i dd| d| d|dt!          | ||          dt          |                    d          pt#          |d                    dt          |                    d          pt#          |d                    dt          |                    d          t$                    r|                    d          nt'          |          d|                                ddd|d|d|d|d d         ddd t+          |           d!i d"|}t-          |          |_        t1          |           t3          |           |S )$Nr  r   r   	timestampedited_timestampauthorglobal_namer  discordmessage_referencer  referenced_messagezdiscord--r   r   r^   r+  r_   r,  r`   r   rd   gQ?ra   rb   rc   r   r-  r.  zdiscord-restr0  r  r1  r   )r   r=   r2   r  r2  r3  r  r   nowr   r   r>   r@   r6  r   r7  r  r8  r?   r9  r:  r  r   r   r   r   )r  r  midr*   r<  r=  r  r   r1  message_ref
referencedra   rb   rc   r@  r   s                   rV   $build_storycard_from_discord_messager  v  s   
dhhtnn"
#
#
)
)
+
+C t &&D
4yy1}}d++}tdhh{++Qtxx8J/K/KQrRRXXZZK%k22Phll8<6P6PJ#-dhhx.@.@$#G#GOTXXhRF&**]++fvzz*/E/EfTXIYIYf]fggmmooGK3=dhhGZ>[>[]a3b3bj$((.///hjK G+//,77=2>>DDFF >7A$((K_B`B`bf7g7goTXX2333mo
*....4"55;;==,T22Ivt!$	7;;F   (j((3((( !z3777 &**W%%Cs)C)CDDD	
 FJJy))G_T3-G-GHHH *4FJJy4I4I4)P)Pk

9%%%VefjVkVk  ))+++ 4 ) v T ete   )...    K!D$ !&&DOtKru   rq  limit_per_channel1tuple[list[StoryCard], dict[str, int], list[str]]c                   g }i }g }| D ]}d}	 t          |||          }	nX# t          $ rK}
|                    | dt          t	          |
                    d d                     d||<   Y d }
~
id }
~
ww xY w|	D ]}t	          |                    d          p|                    d          pd                                          }t          |          }|r||k     ret          ||          }|sy|                    |           |d	z  }|||<   i }|D ]}|||j	        <   t          |                                          }|                    d
 d           |||fS )Nr   )rp  rk   r   rl   r  r  r   )r  r   c                    | j         S r   r   r   s    rV   r   z'collect_discord_cards.<locals>.<lambda>  s    q~ ru   Tr   )r  r4   r   rp   r   r=   r2   r  r  r  r?   valuesrL  )rq  rp  rB  r  rX   statserrorsrz  producedr   excr  r<  r=  r   uniqr   ordereds                     rV   collect_discord_cardsr    s     EEF  	5cN_```HH 	 	 	MMS@@Js3xx$8$8#$>@@AAAE#JHHHH	
  		 		Ddhh{33Ytxx@R7S7SYWYZZ``bbK-k::J j8337MMMD LLMHHc

!#D  QT

4;;==!!GLL--tL<<<E6!!s   "
A7AA22A7c                  "#$ t          t                      dz  i           }t          |t                    r|                    d          ng }g }t          |t
                    rW|D ]S}	 t          |                    dd                                                    |                                 k    rPt          |                    d          pd          }|rt          	                    |          n#t          
                    t          j                  }|j         |                    t          j                  }||k     rt          |                    d          pd          }	|	                    d	          rt!          d
          sd}	|                    t%          dMi dt          |                    d          pd          dt          |                    d          p|           dt          |                    d          pd          dt          |                    d          pd          dt          |                    d          pd          dd |                    dg           D             d d         d|                                dt)          |                    d          pd          dt          |                    d          pd          dt          |                    d          pd          dd |                    dg           D             d d         dt          |                    d          pd          d|	dt          |                    d          pd          dt          |                    d          t                    r|                    d          ni dt)          |                    d          pd          dt          |                    d          pd           d!t          |                    d!          pd          d"t          |                    d"          pd          d#t          |                    d#          pd$          d%t+          |                    d%          pd&          d't+          |                    d'          pd&          d(t+          |                    d(          pd&          d)t-          |                    d)                    d*t/          |                    d*                    d+t          |                    d+          pd          d,t1          |                    d,          d-.          d/t          |                    d/          pd                     D# t2          $ r Y Qw xY wt5          | ||0          }
g }t7          d          D ]%}t9          |           }t;          ||           }|r n&t=          |          |k     rgt?          | tA          |d1z  d2          .          }|rCtC          |          }|D ]1}||v r|"                    |           |                    |           2|st          |t
                    r|D ]}t          |                    dd                                                    |                                 k    r=|                    d          r(|                    t          |d                              g }tC                      }|D ]1}||v r|"                    |           |                    |           2|}d3 |
D             }g "tC                      #tC                      }dN"#fd8}|D ]} ||           |
D ]}|j#        r ||j#                   tA          |d9z  d:          }tA          |dz  d;          }"rt=          |          |k     ro"$                    d<          $$|v r1|"                    $           	 tK          $          |tM          d=>          z
  k     rln# t2          $ r Y yw xY wtO          | $          \  }}}t          |t                    rt          |                    d/          p|                    d?          pd          (                                }t          |                    d@          p|                    dA          pd          (                                                                )                    dB          }|r6||                                 )                    dB          k    r ||           |r$t;          ||           D ]}|$k    r ||           $|v rFtU          $fdC|
D             d           }|r%t          |t                    r|j+        sPt          |                    d/          p|                    d?          pd          (                                |_+        |j,        s;t          |                    d          pd          (                                |_,        t[          t          |                    dD          pd                    }|rCt=          |          t=          t[          |j.        pd                    k    r|d dE         |_.        
|st_          || $dF|  dG$ ||H          } | s-|
                    |            |"                    | j#                   t=          |
          |k    rn"rt=          |          |k     o|r<dI |D             }!|
D ]}||!|j#        <   t          |!0                                          }
tc          |
          }
te          |
          }
|
3                    dJ dKL           |
d |         S )ONzx_intel_feed.jsonrX   r   r   r   r   r.  cacher/  r   r  r   r^   r_   r`   c                n    g | ]2}t          |                                          #t          |          3S r   r   r2   rq   s     rV   rt   z)collect_account_cards.<locals>.<listcomp>  s2     [ [ [ACPQFFLLNN [Q [ [ [ru   rv   rd   g?ra   r{   rb   r}   rc   c                n    g | ]2}t          |                                          #t          |          3S r   r  rq   s     rV   rt   z)collect_account_cards.<locals>.<listcomp>  s1    UUUc!ffllnnUc!ffUUUru   r   r0  r  r   r   template_idcommunity_briefglancetimeline_dateurgencynormalmanual_pickF
manual_pinmanual_bottomre   rf   rg   rh   ri   rj   r1  )r  rB  rC  rh  <   c                *    h | ]}|j         	|j         S r   r  r   s     rV   r   z(collect_account_cards.<locals>.<setcomp>*  s!    ;;;ad;qt;;;ru   r   r   r   r[   c                    t          | pd                                          }|r|v rd S                     |                               |           d S )Nr   )r   r2   rK  r   )r   r;  queuequeueds     rV   enqueue_status_idz0collect_account_cards.<locals>.enqueue_status_id/  s\    %+2$$&& 	cVmmF

3Sru   r   0      r      )days	parent_idreply_to_accountparent_accountrW  c              3  2   K   | ]}|j         k    |V  d S r   r  )rr   rs   r  s     rV   r   z(collect_account_cards.<locals>.<genexpr>W  s/      BB11A1AQ1A1A1A1ABBru   r*   r-  r  r  )r  r  r   r.  
tweet_metac                ,    i | ]}|j         	|j         |S r   r  r   s     rV   
<dictcomp>z)collect_account_cards.<locals>.<dictcomp>u  s#    'N'N'NA'Na'N'N'Nru   c                    | j         S r   r   r   s    rV   r   z'collect_account_cards.<locals>.<lambda>|  rI  ru   Tr   r   )r   r   r   r[   )4	read_jsondata_dirr>   r@   r=   r?   r   r8   r   r   r  r   r   r   r   r  r   r   r7  r:  r5   rv  r   r   r   r4   rQ  rangefetch_profile_pageextract_status_idsr2  (fetch_account_status_ids_from_nitter_rssr6   rJ  rK  r  popr4  	timedeltara  r2   rZ  nextr1  r0  rp   r   parse_status_pager  merge_reply_chain_cardsmerge_numbered_thread_cardsrL  )%r  rB  rC  cached_payloadcached_cards_rawcached_cardsr  	publishedpublished_dtprovider_rawrX   ids_profile_textrss_idsexistingr;  dedupedrO  seen_card_ids	processedr  r   max_fetch_roundsmax_collectedstatus_markdownr.  r  r1  r  rid	meta_textr   r  r  r  r  s%                                     @@@rV   collect_account_cardsr     s   xzz,??DDN6@QU6V6V^~))'222\^$&L"D)) /$ .	 .	D-txx	2..//55778>>;K;KKK 8 8 >B??	DMmx55i@@@S[S_S_`h`lSmSm&.#/#7#7x|#7#L#LL(**"488J#7#7#B7CC**=99 +,yBYBY +#*L##   txx~~3444 #DHHY$7$7$C8 D D D   52666 "$((7"3"3"9r:::	
 !$DHHY$7$7$=2 > > > !\ [)R1H1H [ [ [\^]^\^ _ _ &2%;%;%=%=%= $),)?)?)G4#H#H#H #&dhh{&;&;&Hy"I"I"I  #488H#5#5#@AAA VUdhhvr.B.BUUUVXWXVXYY "%TXXj%9%9%?R!@!@!@ ". %((?(?(E2$F$F$F 8B$((9BUBUW[7\7\ d 3 3 3bd  $),)?)?)F3#G#G#G!" %((?(?(TCT$U$U$U#$  #488H#5#5#;<<<%& '*$((?*C*C*Ir&J&J&J'( !$DHHY$7$7$C8 D D D)* %)-)@)@)IE$J$J$J+, $((>(>(G%#H#H#H-. '+488O+D+D+M&N&N&N/0 %:$((=:Q:Q$R$R$R12 &<DHH^<T<T%U%U%U34 (+4884D+E+E+K'L'L'L56 &<DHH^<T<T\]%^%^%^%^78 %((?(?(E2$F$F$F9   @     B  E C1XX  )(33 x88 	E	 3xx):i"nb))
 
 
  	 3xxH    (??S!!!

3  &-- 	0( 0 0txx	2..//55778>>;K;KKKPTPXPXY]P^P^KJJs4:/// UU 	  	 ChLLNN3;;U;;;MEuuF%%I         # $ $4 	$ad###9q="--	Ar**M
 4C	NN%55599Q<<y  h	$X..(YA=N=N=N2NOO P 	 	 	H	 1FhPX0Y0Y-:j$'' 	/jnn];;`z~~k?Z?Z`^`aaggiiK":>>2D#E#E#oXhIiIi#omoppvvxx~~   A   A   H   H  IL   M   M //8>>3C3C3J3J33O3OOO!!+... 	+)/8DD + +(??%%c***}$$BBBBBBBDIIH 9Jz488 9+ {+.z~~m/L/L/qPZP^P^_jPkPk/qoq+r+r+x+x+z+zH(+ \+.z~~m/L/L/RPR+S+S+Y+Y+[+[H(&s:>>&+A+A+GR'H'HII	 9Y#jARAXVX6Y6Y2Z2Z!Z!Z(1%4%(8H% 	 ===8==!
 
 
  	T$'"""u::&&i  4C	NN%555l  &'N'N'N'N'N 	 	AF14LLV]]__%%#E**E'..E	JJ++TJ:::)s3   -AV-9BV-Q V--
V;:V;5$a 
a('a(c                   t          t          |                     }|sd S t                              |          }|sd S 	 t	          |                    d                    }n# t          $ r Y d S w xY wd|cxk    rdk    rn n|S d S )Nr      )strip_links_mentionsrp   THREAD_PREFIX_REr   r3   r   r4   )r*   srcmidxs       rV   _thread_indexr    s    
z$//
0
0C ts##A t!''!**oo   ttC~~~~2~~~~~
4s    "A# #
A10A1c                    t          t          |                     }t                              d|          }t	          |d          S )Nr   r+  )r  rp   r  subcompact_point)r*   r  s     rV   _thread_seedr    s<    
z$//
0
0C


r3
'
'Cc"""ru   set[str]c                    t          t          |                                                     }t          t                              d|                    }h dfd|D             S )Nz[a-z0-9\u4e00-\u9fff]{3,}>      今天   今晚   分享   我們   更新   活動   這次todayrenaisstonightofficialprotocol	communityc                    h | ]}|v|	S r   r   )rr   rs   stops     rV   r   z%_content_token_set.<locals>.<setcomp>  s    ///!$Aru   )r  rp   r8   rJ  r   findall)r*   r  tokensr  s      @rV   _content_token_setr     sm    
z$//
0
0
6
6
8
8C8#>>??F  D 0///v////ru   abr5   c           
         t          |           }t          |          }|r|sdS t          ||z            t          dt          t          |          t          |                              z  S )Nr   r   )r   r2  r6   r7   )r!  r"  a_setb_sets       rV   _token_overlap_ratior&    sg    q!!Eq!!E  suu}As3u::s5zz'B'B C CCCru   dict[str, int]c                    dddddd}| D ]]}t          |j        t                    r|j        ni }|D ]5}||xx         t          |                    |d          pd          z  cc<   6^|S )Nr   r&  )r>   r  r@   r3   r=   )rX   outr   r  r   s        rV   _sum_metricsr*    s    $%1aZ[\\C 6 6",T\4"@"@H$,,b 	6 	6CHHHGKKQ//41555HHHH	6Jru   r   r7  c           	        t          | d           }g }|D ]c}t          |j        p|j                  }|r| dnd}t	          |j        p|j        p|j                  }|r|                    | |            dd                    |          d d         }|d         }|d         }	t                      }
g }|D ]}|j	        pg D ]k}t          |pd                                          }|r||
v r,|
                    |           |                    |           t          |          d	k    r nlt          |          d	k    r nd}|D ]}|j        r	|j        } nt          di d
|j        d|j        d|j        d|j        d|	j        p|j        d|	j        p|j        d d         d|	j        dt+          d |D                       d|	j        d|	j        d|r
|d d         n|	j	        p	|j	        pdgd|d|	j        p|j        d|dt3          |          dt          |j        pd          }t+          d |D                       dz   |_        t9          |           t;          |           |S ) Nc                r    t          | j                  p#t                              t          j                  S r   r  r   r   r  r   r   r   s    rV   r   z%_merge_thread_group.<locals>.<lambda>  s&    ';AN'K'K'ix||\d\hOiOi ru   r   z/ r   r]   iX  r   r   r  r   r   r^   r_   r`   rv   r   rd   c              3  B   K   | ]}t          |j        pd           V  dS r   N)r5   rd   r   s     rV   r   z&_merge_thread_group.<locals>.<genexpr>  s1      @@auQ\0S11@@@@@@ru   ra   rb   rc   u   觀點r   r.  r0  r  r1  c              3  B   K   | ]}t          |j        pd           V  dS r1  )r5   r   r   s     rV   r   z&_merge_thread_group.<locals>.<genexpr>  s1      EE1E!,"5#66EEEEEEru   g?r   )r   r  r   r^   rp   r_   r   r   rJ  rc   r   r2   rK  r2  r0  r7  r  r   r   r`   r   r6   ra   rb   r.  r*  r1  r   r   r   )r   rowsr  r   r  prefixr  
merged_rawrO   lasttag_seenrc   tagtr?  r  s                   rV   _merge_thread_groupr:    sD   %iijjjDE , ,DM7TZ88"*C$-E4<E4:FF 	,LLF*D**+++U##ETE*JGE8DHD 
 
9? 	 	CCI2$$&&A XLLOOOKKNNN4yyA~~ t99>>E E   	$EE	    88 II kk	
 - .33 && @@4@@@@@@ .. {{ JT"1"XXDI$I$Iz  0%. E T"""  )/R000!F$ EEEEEEEKF   vMru   c                ~   | s| S i }| D ]B}|                     |j                                        g                               |           Cg }t	                      }|                                D ]\  }}d |D             }t          |          dk     r&d |D             dfddfd}|D ]L}|j        }	t          |j	        pd          
                                }
|	r|
r|
|	k    r<|
|v r ||	|
           Mi }|                                D ]9\  }	} |	          }|                     |g                               |           :|                                D ]}t          |          dk     rt          |d           }t          |d         j                  p#t                              t"          j                  }t          |d         j                  p|}||z
                                  dk    rt)          |          |                               |D ]}|                    |j                   |s| S d |D             }g }t	                      }| D ]w}|j        |vr|                    |           !|                    |j                  }|>|j        |v rH|                    |           |                    |j                   x|D ]Wj        |v rt/          fd|D                       r(|                               |                    j                   X|S )Nc                ,    i | ]}|j         	|j         |S r   r  r   s     rV   r  z+merge_reply_chain_cards.<locals>.<dictcomp>  s#    'F'F'FA'Fa'F'F'Fru   r  c                    i | ]}||S r   r   )rr   r;  s     rV   r  z+merge_reply_chain_cards.<locals>.<dictcomp>  s    %A%A%A3c3%A%A%Aru   rs   r   r   c                j    |          | k    r%|                   | <   |          } |          | k    %| S r   r   )rs   
dsu_parents    rV   findz%merge_reply_chain_cards.<locals>.find  sE    Q-1$$ *:a= 9
1qM Q-1$$ Hru   r!  r"  r[   c                N     |           } |          }||k    r||<   d S d S r   r   )r!  r"  rarbr?  r@  s       rV   unionz&merge_reply_chain_cards.<locals>.union  s:    aBaBRxx!#
2 xru   r   c                r    t          | j                  p#t                              t          j                  S r   r-  r   s    rV   r   z)merge_reply_chain_cards.<locals>.<lambda>  s)    2Fq~2V2V2tZbZfZfgogsZtZt ru   r.  r   r/  i c                    i | ]
}|j         |S r   r  )rr   r  s     rV   r  z+merge_reply_chain_cards.<locals>.<dictcomp>#  s    222AD!222ru   c              3  8   K   | ]}|j         j         k    V  d S r   r  rr   r   r  s     rV   r   z*merge_reply_chain_cards.<locals>.<genexpr>5  s,      ..Qqtvy ......ru   )rs   r   r   r   )r!  r   r"  r   r   r[   )
setdefaultr   r8   r   rJ  r  r2  r  r   r1  r2   r  r   r  r   r   r  r   r   total_secondsr:  rK  r=   any)rX   
by_accountr   merged_cards
member_ids_accountr3  id_maprD  r;  r  groupsrootr   r  dt_mindt_maxrowmerged_by_idr)  inserted_idsreplacementr?  r@  r  s                         @@@rV   r  r    s    -/J E Edl0022B77>>tDDDD$&L55J$**,, ,' ,'$'F'F'F'F'Fv;;??%A%A&%A%A%A
	 	 	 	 	 		$ 	$ 	$ 	$ 	$ 	$ 	$  	& 	&D'CD,23399;;I i 9+;+;F""c9%%%-/ 	5 	5IC499DdB''..t4444]]__ 	' 	'E5zzA~~U(t(tuuuG)'!**ABB`hllS[S_F`F`F)'"+*BCCMvF..00=@@(11F''' ' 'sv&&&&'	'  22\222LC UUL 
) 
)7*$$JJt"&&tw//>\))

;(((( $ $9$$....#..... 	

6####Jru   r   rows_sortedpivot	directionused_ids	max_hoursc                   | sd S 	 |                      |          }n# t          $ r Y d S w xY w|dvrd S t          |j                  p#t                              t          j                  }t          |j	        p|j
                  }t          |j	        p|j
        |j                  }||z   }	d|	cxk    rt          |           k     r.n n*| |	         }
|	|z  }	|
j        |v r6t          |
j	        p|
j
                  rRt          |
j                  p#t                              t          j                  }t!          ||z
                                            |dz  k    rt          |
j	        p|
j
                  }t          |
j	        p|
j
        |
j                  }t%          ||          }t'          |          t'          |          k    }|dk    s|r|
S d|	cxk    rt          |           k     (n d S )N>   r   r/  r   i  g
ףp=
?)index
ValueErrorr  r   r   r  r   r   r  r   r^   infer_topic_phrasera   r2  r  r  absrJ  r&  
dedupe_key)rY  rZ  r[  r\  r]  r  base_dt	base_seed
base_topicpr   cand_dt	cand_seed
cand_topicoverlaptopic_matchs                   rV   _find_neighbor_non_indexedrm  <  s     t&&   ttt"5#566T(,,x|:T:TGU^:u{;;I#EN$Aek5?SSJiA
q
#
#
#
#3{##
#
#
#
#
#1~	Y7h4$*55 	&t'899WX\\(,=W=W'!002233i$6FFF !<$*==	'(CT^TT
&y)<< ,,
:0F0FFd??k?K q
#
#
#
#3{##
#
#
#
#  4s    
**c                	   !"# | s| S i }| D ]B |                      j                                        g                                           Ct	                      }g }|                                D ]h\  }}t          |d           }t          |          }d}||k     r8||         }	t          |	j	        p|	j
                  }
|
s|dz  }2t          |	j                  p#t                              t          j                  }t#          |	j	        p|	j
                  }t%          |	j	        p|	j
        |	j                  }|	g}|
}|dz   }||k     rx||         }t          |j	        p|j
                  }|s|dz  }2t          |j                  p#t                              t          j                  }t)          ||z
                                            dk    r|dz  }t#          |j	        p|j
                  }t%          |j	        p|j
        |j                  }t-          ||          }t/          |          t/          |          k    }t)          ||z
            dk    pt)          ||
z
            dk    }|dk    s|r;|r9|                    |           t1          ||          }t          |          dk    rn|dz  }||k     x|rt          |d	           }|d         }|d
         }t3          ||d
|          "t3          ||d|          #"r0t5          "fd|D                       r|                    "           #r0t5          #fd|D                       r|                    #           t          |          dk    rCt7          |          !|                    !           |D ]}|                    |j                   |dz  }||k     8j|s| S g }t	                      }| D ]x  j        |v rX j        |v rt=           fd|D             d           }|/|                    |           |                     j                   c|                                y|D ]U!!j        |vrJt5          !fd|D                       r/|                    !           |                    !j                   V|S )Nc                r    t          | j                  p#t                              t          j                  S r   r-  r   s    rV   r   z-merge_numbered_thread_cards.<locals>.<lambda>o  s)    1Ean1U1U1sYaYeYefnfrYsYs ru   r.  r   r   i  rh  g?ri   c                r    t          | j                  p#t                              t          j                  S r   r-  r   s    rV   r   z-merge_numbered_thread_cards.<locals>.<lambda>  s+    ;OPQP^;_;_;}ckcocopxp|c}c} ru   r/  )r[  r\  c              3  8   K   | ]}j         |j         k    V  d S r   r  )rr   r   root_neighbors     rV   r   z.merge_numbered_thread_cards.<locals>.<genexpr>  -      (Q(Qa)9QT)A(Q(Q(Q(Q(Q(Qru   c              3  8   K   | ]}j         |j         k    V  d S r   r  )rr   r   tail_neighbors     rV   r   z.merge_numbered_thread_cards.<locals>.<genexpr>  rs  ru   r  c              3  <   K   | ]}|j         j         k    |V  d S r   r  )rr   r  r   s     rV   r   z.merge_numbered_thread_cards.<locals>.<genexpr>  s+      KKa1447??????KKru   c              3  8   K   | ]}|j         j         k    V  d S r   r  rH  s     rV   r   z.merge_numbered_thread_cards.<locals>.<genexpr>  s,      0P0Pq1B0P0P0P0P0P0Pru   )rI  r   r8   r   rJ  r  r   r2  r  r   r^   r  r   r   r  r   r   r  ra  ra   rb  rJ  r&  rc  r6   rm  allr:  rK  r  r  )$rX   rL  member_ids_in_mergedrM  rO  r3  rY  nibasebase_idxrd  re  rf  r   last_idxjr   cand_idxrh  ri  rj  rk  rl  index_relatedgroup_sortedheadtailr   r)  rW  rX  r   r  rr  ru  s$                                   @@@@rV   r  r  c  s>    -/J E Edl0022B77>>tDDDD%(UU$&L$**,, : :$T's'sttt!eeq>D$T]%@djAAH Q*4+<==[hlA[A[G$T]%@djAAI+DM,GTZXXJFEHAAa%%"1~()D$*EE FA.t/@AA_X\\RZR^E_E_')88::;;iGGFA()D$*EE	/0KT^\\
.y)DD(44
:8N8NN8+,,2 68h.//25  sNNkN}NLL&&&"8X66H5zzQQ/ a%%2  	0%e1}1}~~~#A#B' :;XZey z z z :;XYdx y y y  0S(Q(Q(Q(Q5(Q(Q(Q%Q%Q 0LL///  0S(Q(Q(Q(Q5(Q(Q(Q%Q%Q 0LL///5zzQ,U33##F+++ 3 3A(,,QT2222FAm !eep  C UUL 	 	7***w,&&KKKK<KKKTRRK&

;'''  )))

4 ( (9L((S0P0P0P0PC0P0P0P-P-P(JJvVY'''Jru   	input_urlc                $   | pd                                 }|st          d          |                    d          rd|t          d          d          z   }|                    d          sd|z   }t	          |          }|j        dvrt          d          |j                             d          }t                              d|           }|st          d	          |	                    d
          |	                    d          }}d| d| S )Nr   z	empty URLzhttp://zhttps://>   x.com	www.x.comtwitter.comwww.twitter.comz)only x.com/twitter.com URLs are supportedr  r  invalid tweet URLr   r  r  )
r2   r`  r  r2  urlparsenetlocpath	STATUS_REr   r   )r  r   r   r  r   r  r  s          rV   normalize_x_urlr    s   ?
!
!
#
#C &%%%
~~i   13s9~~//00>>*%% 3c]]F}TTTDEEE;S!!D4d4455E .,---QQhH8H88h888ru   	tweet_urlc                   t          |           }t                              |          }|st          d          |                    d          |                    d          }}t          ||          \  }}}|st          d          t          ||||||          }	|	st          d          |rt          |	g|t                                 |	S )Nr  r   r  zunable to fetch tweet)r.  r  zunable to parse tweet)rZ   )
r  r  r   r`  r   ra  rA   r  r   feedback_training_text)
r  r	   
normalizedr   r  r  r  r.  r  r   s
             rV   build_card_from_urlr    s     ++JZ((E .,---QQhH,A(H,U,U)OXz 42333  D  42333 _"D67E[E]E]^^^^Kru   r  PathdefaultAnyc                    |                                  s|S 	 t                              |                     d                    S # t          $ r |cY S w xY w)Nzutf-8)encoding)existsr$   r   	read_textr4   )r  r  s     rV   r  r    s`    ;;== zz$..'.::;;;   s   -A AAr   )r   r   r	   r   r
   r   r   r   )r   )rX   rY   r	   r   rZ   r   r   r[   )r*   r   r   r   )
rX   rY   r   r   r   r   r	   r   r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r  )r  r   r  r   r   r  )r  r   rB  r   rC  r3   r   rY   )rP  r   r   r   )r  r   r  r   r   rU  )r   r   )r  r   r   r   )r  r   r  r   r  r   r   r   )r  r   rp  r   rk   r3   r   r  )r  r   r  r   r   r  )
rq  r   rp  r   rB  r   r  r3   r   r  )r*   r   r   r   )r*   r   r   r   )r*   r   r   r  )r!  r   r"  r   r   r5   )rX   rY   r   r'  )r   rY   r   r7  )rX   rY   r   rY   )r   )rY  rY   rZ  r7  r[  r3   r\  r  r]  r3   r   r  )r  r   r   r   )r  r   r	   r   r   r7  )r  r  r  r  r   r  )+
__future__r   r   r   
_bootstrapr   
_editorialglobalsupdatevarsrW   r   r   r   r   r  r  rA  DEFAULT_MAX_POSTS_PER_ACCOUNTrQ  r   ra  r}  r  r  r  ru  r  r  r  r   r  r  r   r&  r*  r:  r  rm  r  r  r  r  r   ru   rV   <module>r     sv   " " " " " " % % % % % % % % % % % % 		  j!! " " " 		  j!! " " "L" L" L" L" L"^f f f f fR   . 	/ / / / /d   8! ! ! !H+ + + +\= = = =F 34 4 4 4 4n   " " " "(% % % %P' ' ' ',   2I I I I Nk 4 4 4 4 4", , , ,f ;	$" $" $" $" $"N Ol q q q q qh    # # # #0 0 0 0D D D D   3 3 3 3lR R R Rt $ $ $ $ $NZ Z Z Zz9 9 9 9*    6     ru   