o jz@sddlmZddlZddlZddlZddlZddlmZmZmZddl m Z ddl Z ddl m ZddlmZmZmZmZmZmZddlmZmZmZmZdd lmZmZdd lmZdd l m!Z!dd l"m#Z$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+ddl,m-Z-m.Z.ddl/m0Z0m1Z1m2Z2ddl3m4Z4ee5Z6e dZ7dZ8ej9:ej9;e<dZ=dZ>dZ?dZ@dZAdZBdZCdZDddZEddZFddZGdd ZHd!d"ZId*d$d%ZJejKd&d'd(d)ZLdS)+) getLoggerN)datetimetimezone timedelta) load_dotenv) celery_app) create_deals_table_if_not_existsload_watermarksave_watermark to_db_keysupsert_deal_rowsDB_PATH)fetch_deals_for_dealsfetch_product_batch validate_asinfetch_current_stats_batch) TokenManagerTokenRechargeError) FUNCTION_LIST)get_seller_info_for_single_deal) load_settingscalculate_all_in_costcalculate_profit_and_margincalculate_min_listing_price)get_1yr_avg_sale_priceget_percent_discount get_trend)classify_seasonalityget_sells_period)_process_single_dealclean_numeric_values_process_lightweight_update)get_db_connectionz3.0-Consolidateddealsz headers.json2smart_ingestor_lockicCs(tdddtjd}|t|d}|S)zFConverts Keepa time (minutes since 2011-01-01) to ISO 8601 UTC string.rtzinfo)minutes)rrutcr isoformat) keepa_minutes keepa_epoch dt_objectr25/var/www/agentarbitrage/keepa_deals/smart_ingestor.py_convert_keepa_time_to_iso0sr4cCsD|sdSt|tj}tdddtjd}||}t|dS)zIConverts an ISO 8601 UTC string to Keepa time (minutes since 2011-01-01).rr)rr*<)r fromisoformat astimezonerr-int total_seconds)iso_strr1r0deltar2r2r3_convert_iso_to_keepa_time6s r<c Csz+t|tj}ttj}||tddkr*td|d| | }Wnt yE}zt d|WYd}~nd}~wwt |dS)z5Saves watermark with protection against future dates.hoursz#Clamping Extreme Future Watermark: z -> z!Error checking watermark safety: N) rr6r7rr-nowrloggerwarningr. Exceptionerrorr ) iso_timestampwm_dtnow_utcer2r2r3save_safe_watermark?s  rIc Cs||sdS|dg}|dg}|dg}d}t|dkr)|ddkr)|d}nt|dkr9|ddkr9|d}|dkr?dSg}t|dkrT|ddkrT||dt|dkrg|ddkrg||dt|d krz|d dkrz||d t|d kr|d dkr||d t|d kr|d dkr||d t|dkr|ddkr||dt|dkr|ddkr||dt|d kr|d dkr||d t|d kr|d dkr||d t|d kr|d dkr||d |sdS|d d}|dkr|dkrdSt|}|d kr!dS||dkr*dS|dkr<|||}|dkr 1 hour and re-queues them. rr>z SELECT r.asin FROM user_restrictions r JOIN deals d ON r.asin = d.asin WHERE r.is_restricted IS NULL AND d.last_seen_utc < ? cSg|]}|dqS)rr2.0rowr2r2r3 z.requeue_stuck_restrictions..z!Ghost Restriction Sweeper: Found z stuck ASINs. Re-queueing.4keepa_deals.sp_api_tasks.check_restriction_for_asinsargsz7Ghost Restriction Sweeper: No stuck restrictions found.Nz%Error in requeue_stuck_restrictions: )r#rcursorrr@rr-rr.executefetchallrAinforUcelery send_taskrCrD)connrhcutoffqueryasinsrHr2r2r3requeue_stuck_restrictionss$   &rrrPc szI|jdkrtd|jdWdSg}tt+}tj|_|}t t j t dd}d}||||f|}Wdn1sHwY|sRWdSdd |D}td t|d ||d t|td }t||ddd\} } } } | r|| | rd| vrtdWdSg} tt } t| }Wdn1swY| dD]]}|dtfdd|Dd}|sqt|}t||}|rt |}t t j |d<d|d<|d}|dur t!|t"t#fr |dkr tdd|dd| $|q| rHtt#}|}t%|| ||&tdt| d WdWdS1s@wYWdSWdSt'yh}ztj(d!|d"d#WYd}~dSd}~ww)$z Finds deals that are approaching the Janitor's 72h deadline (e.g. > 48h old) and forces a refresh to prevent them from being deleted if they are still valid. z(Skipping Stale Rescue: Low Refill Rate (z/min).N0r>z SELECT * FROM deals WHERE last_seen_utc < ? ORDER BY last_seen_utc ASC LIMIT ? cSr_ASINr2r`r2r2r3rcrdz&rescue_stale_deals..zStale Deal Rescue: Found z deals > 48h old. Refreshing:  KEEPA_API_KEYrPdaysoffersproductsz.Stale Deal Rescue: Keepa returned no products.asinc3s |] }|dkr|VqdS)rvNr2rarr~r2r3 sz%rescue_stale_deals.. last_seen_utc stale_rescuesourceProfitrzStale Rescue: ASIN z updated but Profit is now $z.2fz'. It will be hidden from the dashboard.z*Stale Deal Rescue: Successfully refreshed  deals.zError in rescue_stale_deals: Texc_info))REFILL_RATE_PER_MINUTErArkr#rsqlite3Row row_factoryrhrr@rr-rr.rirjrUrequest_permission_for_callosgetenvrupdate_after_callrBopen HEADERS_PATHjsonloadrTnextdictr"r! isinstancer8floatrVr commitrCrD) token_managerlimit stale_rowsrnrhrorp stale_asinsapi_key prod_resp_ tokens_leftrows_to_upsertf headers_listp existing_rowexisting_row_dict processed_rowprofitrHr2rr3rescue_stale_dealssr              $   ( "rzkeepa_deals.smart_ingestor.run)namec<s tjtjj}|jttd}|j ddst ddSzzVt dt dt ttd}td}|sSt d WW|rQ|t d dSdSt|}|rat d n|t|d d t d|dt}|jd krd}t d|dt}|durt dttjt dd!}t d|t"|nPz5t#|$tj}ttj}||t ddkrt %d|d|!d|t dd!}t"|Wnt&y} zt d| WYd} ~ nd} ~ wwt'|} t d|d| dg} d} d} t d |(| t)kr,t %d!t)d"nt*| |krAt %d#t*| d$d } n|dd}d%}t+|D]R}z#t,| |d&|d'\}}}|durf|-||rqd(|vrqWn0WqNt&y} z t %d)| d*|d+d,| t./d-|d+WYd} ~ qNd} ~ ww|rd(|vrnT|d(d.st d/nGd0d1|d(d.D}|j0d2d3d d4d}|D]}|d5| krt d6|1d7d8d }n| 2|q|rn | d+7} t./d+q| st d9WW|r|t d dSdSt d:t*| d;| j0dd1| D}t3i}d}zz]t4t5d?d}t6j7|_8|9}d@:dAt*|}|;dBt<dC|d||=}|D]0}|dD}|dE}|dF}d} | rt dG|dHdIqg>|dHt?|||dH<qgWnt&y} zt %dJ| WYd} ~ nd} ~ wwW|durz|@Wn0t&yt j%dKd dLYn wn|durz|@Wwt&yt j%dKd dLYwwwtA}!|jdMkr d+}!t dN|jdO|!dPn+|jd kr d }!t dQ|jdO|!dRn|jdSkr5d-}!t dT|jdO|!dRtBtC }"tDE|"}#Wdn 1sJwYd}$t+dt*| |!D] }%|(| |%|%|!}&dUd1|&D}'fdVd1|'D}(fdWd1|'D})i}*g}+|(r|dXt*|(tF||(dYd dZ\},}}}|r|-||,rd[|,vr|,d[D]}-tG|-1d\r|+2|-d7qt d]|-1d7q|+rt+dt*|+tHD]?}.|+|.|.tH}/|d t*|/tI||/dYd+d d^\}0}}}|r|-||0rd[|0vr|0d[D] }-|-|*|-d7<qq|)rR|dt*|)tF||)d_d dZ\}1}}}|r<|-||1rRd[|1vrR|1d[D] }-|-|*|-d7<qHg}2|&D]h}|d7}3|3|*vrcqV|*|3}4|4J|d}5|3vrtK||3|4}5|5rtL|5}5ttj!|5d`<da|5db<n%tM|4||}6tN|4|6|}5|5rtL|5}5tO|5}5ttj!|5d`<dc|5db<|5r|22|5qV|2rNt ddt*|2dedfd1|2ddMDdgzVt4t5d?dD}7|79}8tP|8|2|#|7Q|$t*|27}$dhd1|2D}9|9rtjRdi|9gdj|&dk}:tS|:d5};tT|;t dl|;Wdn 1s(wYWqYt&yM} zt jdm| d dLWYd} ~ qYd} ~ ww|&dk}:tS|:d5};tT|;t dn|;qYt dot*| dp|$dWn.tUy} z!t %dq| drWYd} ~ W|r|t d dSdSd} ~ wwW|r|t d dSdS|r|t d ww)sN)timeoutF)blockingzD--- Task: smart_ingestor is already running. Skipping execution. ---z+--- Task: smart_ingestor started (Version: z) ---rx XAI_TOKENz KEEPA_API_KEY not set. Aborting.z+--- Task: smart_ingestor lock released. ---z6Skipping sync (Recharge active, estimated tokens low).rP)rzStep 1: Initializing Sync...r%r&z-Low Refill Rate. Reducing NEW_DEALS limit to .zYCRITICAL: Watermark not found. Assuming fresh start/reset required but not handling here.r=r>zWatermark defaulted to z-CRITICAL: Extreme Future Watermark Detected (z > z1). Resetting to 24h ago to restore functionality.zError checking watermark date: zLoaded watermark: z (Keepa time: )rz4Step 2: Paginating through deals to find new ones...Tz/Safety Limit Reached: Stopped pagination after z pages.zNew Deal Limit Reached: Found rrw) sort_typerr$zFetch failed on page z (Attempt rz): drz!No more deals found (empty list).cSsg|] }t|dr|qSr)rrTradr2r2r3rcszrun..cS|dSN lastUpdater2xr2r2r3zrun..)keyreverserzStop Trigger: Deal r~z <= Watermark.zNo new deals found.zFound z? new deals. Sorting by Oldest First for incremental processing.cSrrr2rr2r2r3rr)rcSr_rr2rr2r2r3rcrdr5,?zSELECT * FROM z WHERE ASIN IN (List_at1yr_AvgrzASIN rvz6: Detected as ZOMBIE/BAD DATA. Forcing heavy re-fetch.z Failed to check existing ASINs: z/Failed to close existing-ASIN check connection.rrszCritically Low Refill Rate (z#/min). Reducing SCAN_BATCH_SIZE to z (Max potential cost 22 tokens).zLow Refill Rate (z to prevent Deficit Lockout.zMid Refill Rate (cSr_rr2rr2r2r3rcrdcsg|]}|vr|qSr2r2raaexisting_asins_setr2r3rc csg|]}|vr|qSr2r2rrr2r3rc rrNimrzr}rXzPeek Rejected: ASIN )r{historyr|ryrsmart_ingestor_lightrsmart_ingestorz Upserting z deals to DB. ASINs: cSsg|]}|dqSru)rTrr2r2r3rcZsz...cSsg|] }d|vr|dqSrur2r`r2r2r3rchsrerfrMzWatermark ratcheted to z Chunk processing/upsert failed: z'Watermark advanced (no deals saved) to zTask Complete: Processed z scanned, upserted z--- Task Paused: z$. Releasing lock to free worker. ---)VredisRedisfrom_urlrlconf broker_urllockLOCK_KEY LOCK_TIMEOUTacquirerArkSMART_INGESTOR_VERSIONrrr rrrDlockedreleasershould_skip_sync sync_tokensrrMAX_NEW_DEALS_PER_RUNrr rr@rr-rr.r r6r7rBrCr<emit_heartbeatMAX_PAGES_PER_RUNrUrangerrtimesleepsortrTrVsetr#rrrrrhjoinri TABLE_NAMErjaddrcloseSCAN_BATCH_SIZErrrrrr^COMMIT_BATCH_SIZErupdater"r!rr r r rrmr4rIr)< redis_clientrr xai_api_keyrcurrent_max_deals watermark_isorFrGrHwatermark_keepa_time all_new_dealspagehit_new_deal_limit deal_responsemax_page_retriesattemptrr deals_on_pagefound_older_dealdeal asin_listexisting_rows_map conn_checkc_check placeholdersrowsrlist_atyr_avgr is_zombiecurrent_batch_sizerheaderstotal_upsertedi chunk_deals chunk_asinschunk_new_asinschunk_existing_asinschunk_productsnew_candidates peek_resprj sub_batchrprod_resp_lightrr~ product_datarseller_data_cachernrh new_asinslast_deal_in_chunk new_wm_isor2rr3run(s     L                  :  V                  .   "      r)rP)Mloggingrrrrrrrrdotenvrrworkerrrldb_utilsr r r r r r keepa_apirrrrrrrfield_mappingsr seller_inforbusiness_calculationsrbusiness_load_settingsrrr new_analyticsrrrseasonality_classifierrr processingr r!r"keepa_deals.db_utilsr#__name__rArrpathrdirname__file__rMAX_ASINS_PER_BATCHrrrrrrr4r<rIr^rrrtaskrr2r2r2r3sN        V # a