o
    jEi7                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m	Z	 ddl
mZmZmZmZ ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ d dlZddlmZ dZdd ZdddZ dd Z!dS )    N)load_dotenv)datetime   )fetch_deals_for_dealsfetch_product_batchvalidate_asinfetch_seller_data)TokenManager)FUNCTION_LIST)_process_single_deal)infer_sale_eventsanalyze_sales_performance)get_all_seller_info)load_settingsd   c              
   C   s   t jt jt jtdd}z"t|d}tj| |dd W d   W dS 1 s+w   Y  W dS  t	yL } zt
d|  W Y d}~dS d}~ww )z#Helper to write to the status file...scan_status.jsonw   )indentNzError writing status file: )ospathjoindirnameabspath__file__openjsondumpIOErrorprint)status_dataSTATUS_FILEfe r%   2/var/www/agentarbitrage/keepa_deals/Keepa_Deals.pyset_scan_status   s    &r'   Fdatac           2         sl  t tt  td}t }dd }zdt 	ddd d d	t
jjd
}t| tj|dd tj|d td}	t|	dt d W d   n1 s\w   Y  t| }
dR fdd	}d g }d}d}	 d| d |
j|d t|| \}}}|
| |sd| d n)|di dg }|sd| d n|| |rt||krn|d7 }qz|}|dur|d| n|}|du r|}|d t| d!t| dt|dd" |s|g g dd# tg  |d$d%d& W dS d'd( |D fd)d(tdtt D }|
!  d*}t| }t"#td+ }|| }d,t  d-|  d.|
j$  |
%|swd/| d0|
j$ d1}| |d2|d& W dS i }|D ];}d3d( |D }|
jt|| d t&| |dd4d5\}}}}|
| |rd6|v r|d6 D ]	}|||d7 < qq{d8d9 |' D } i }!| r!t(| }"t"#t|"d+ }#|
%|#s)d:|# d0|
j$ d; n;tdt|"d<D ]2}$|"|$|$d<  }%|
jt|%d+ d t*| |%\}&}}}|
| |&rd=|&v r|!+|&d=  qd> |, D ];\}'}(zt-|(\})}t.|(|)}*|*|(d?< W q* t/ye }+ zjd@|' dA|+ ddB i |(d?< W Y d}+~+q*d}+~+ww dC dD t0 },g }-d}.|D ]l}/|/d7 }'||'}0|0s|-1|'dEdF qyz|0+|/ t2|0|!|dG}1|1r|-1|1 W n+ t/y }+ zjd@|' dH|+ ddB |-1|'dI|+ dF W Y d}+~+nd}+~+ww |.d7 }.|.dJ dkr|dK|.i qydL dM ||-| t|- dN |d$dOd& W dS  t/y5 }+ zjdPt3|+ ddB |d2dQt3|+ d& W Y d}+~+dS d}+~+ww )SzL
    Main script to run the Keepa deals fetching and processing script.
    	XAI_TOKENc              	   S   s   i }t jt jt jtdd}t j|rAzt|d}t	|}W d    n1 s.w   Y  W n t
tjfy@   Y nw ||  t| d S )Nr   r   r)r   r   r   r   r   r   existsr   r   loadr   JSONDecodeErrorupdater'   )status_dictcurrent_statusr"   r#   r%   r%   r&   _update_cli_status3   s    
z,run_keepa_script.<locals>._update_cli_statusRunningz%Y-%m-%dT%H:%M:%S.%fNZz'Worker has started processing the scan.)status
start_timemessagetask_idT)exist_okzKeepa_Deals_Export.csvzkeepa_deals/headers.jsonzLoaded headers: z fieldsFc           	         s,   dt| dt|  d t|t| kr*|s*dt| dt|  d t dddd	R}t|}| |rO|d
gdgtd    n.| D ]+}g }D ]}||d}|dkrq|dkrq|d| d qW|| qW|| qQW d    n1 sw   Y   d   d S )Nz%Entering write_csv. Number of deals: z. Number of rows: .z%Mismatch in write_csv: len(deals) is z but len(rows) is r    zutf-8)newlineencodingzNo deals fetched-r   ASINz=""zCSV written: )	infolenwarningr   csvwriterwriterowgetappend)	rowsdeals
diagnosticr#   rE   rowrow_to_writeheadervalue)CSV_PATHHEADERSloggerr%   r&   	write_csvQ   s&     

 z#run_keepa_script.<locals>.write_csvzStarting Keepa_Deals script...r      zFetching deals page ...)estimated_costzFailed to fetch deals for page r:   rJ   drzNo more deals found on page r   zFound z total deals. Processing )r7   total_dealsprocessed_deals)rK   	CompletedzNo deals found.)r5   r7   c                 S   s   g | ]}t |d r|qS asin)r   rG   .0dr%   r%   r&   
<listcomp>   s    z$run_keepa_script.<locals>.<listcomp>c                    s   g | ]
} ||t   qS r%   )MAX_ASINS_PER_BATCH)r^   i)valid_deals_to_processr%   r&   r`      s       g      Y@z)PRE-FLIGHT TOKEN CHECK: Deals to process=z8PRE-FLIGHT TOKEN CHECK: Calculated total_estimated_cost=z;PRE-FLIGHT TOKEN CHECK: Token manager balance BEFORE check=z2Insufficient tokens for full run. Estimated Cost: z, Available: z. Aborting task.Failedc                 S   s   g | ]}|d  qS r[   r%   r]   r%   r%   r&   r`             )historyoffersproductsr\   c                 S   s:   h | ]}| d g D ]}t|tr
| dr
|d q
qS )ri   sellerId)rG   
isinstancedict)r^   pofferr%   r%   r&   	<setcomp>   s    
z#run_keepa_script.<locals>.<setcomp>z1Insufficient tokens for seller data fetch. Cost: z. Skipping seller info.r   sellersz+Starting Stage 1: Analytics Pre-calculationanalytics_cachezASIN z$: Analytics pre-calculation failed: exc_infozFinished Stage 1.z&Starting Stage 2: Main Data ProcessingzProduct data not found)r?   Title)seller_data_cachexai_api_keyz3: Critical error in main processing loop for deal: zProcessing Error: 
   rY   zFinished Stage 2.z$Starting Stage 3: Saving to DatabasezFinished Stage 3.zScan completed successfully.zMain failed: zAn error occurred: )F)4logging	getLogger__name__r   r   getenvtimer   utcnowstrftimerun_keepa_scriptrequestidr'   makedirsr   r   r   r   r,   debugrB   r	   rA   request_permission_for_callr   update_after_callerrorrG   extendsave_to_databaserangera   refillmathceiltokenshas_enough_tokensr   valueslistrC   r   r.   itemsr   r   	Exceptionbusiness_load_settingsrH   r   str)2api_keyno_cache
output_dir
deal_limitstatus_update_callbackXAI_API_KEYscan_start_timer1   initial_statusr#   token_managerrS   	all_dealspageTOKEN_COST_PER_DEAL_PAGEdeal_response_tokens_left
deals_pagerJ   deals_to_processasin_batchesCOST_PER_PRODUCTproduct_fetch_costseller_fetch_costtotal_estimated_cost	error_msgall_fetched_products_mapbatchbatch_asinsproduct_data_responsern   unique_seller_idsrv   seller_id_listseller_costrb   	batch_idsseller_datar\   productsale_eventsanalysis_resultsr$   business_settings
final_rowsprocessed_countdealproduct_dataprocessed_rowr%   )rP   rQ   rR   rc   r&   r   *   s(  
















"



&r   c              
      s  dd l }dd l tjtjtjtdd}d}|d| d zz|	|}|
 } fddfd	d
|D }|d|  g }	|D ]4}
d}d|
v sad|
v sad|
v sad|
v sad|
v rdd}nd|
v spd|
v spd|
v rrd}|	d|
 d|  qId| dd|	 d}|| ddd |D }ddgt| }d | d!| d"| d}g }| D ]fd#d
|D }|t| q||| |  W n ty } z|jd$| d%d& W Y d }~nd }~ww W |r|  d S d S |r|  w w )'Nr   r   zdeals.dbrJ   zConnecting to database at rU   c                    s.     dd| ddddddddS )	Nz[^a-zA-Z0-9_]r;    r   r:   r>   %Percent)subreplace)name)rer%   r&   sanitize_col_name'  s   .z+save_to_database.<locals>.sanitize_col_namec                    s   g | ]} |qS r%   r%   r^   h)r   r%   r&   r`   *  rf   z$save_to_database.<locals>.<listcomp>zDROP TABLE IF EXISTS TEXTPriceFeeMarginr   ProfitREALRankCountDropsINTEGERr@   z" zCREATE TABLE z( (id INTEGER PRIMARY KEY AUTOINCREMENT, z, )c                 s   s    | ]	}d | d V  qdS )r@   Nr%   r   r%   r%   r&   	<genexpr>:  s    z#save_to_database.<locals>.<genexpr>?zINSERT INTO z (z
) VALUES (c                    s   g | ]}  |qS r%   )rG   r   )row_dictr%   r&   r`   @  s    zDatabase error: Trs   )sqlite3r   r   r   r   r   r   r   rA   connectcursorexecuterH   rB   tupleexecutemanycommitr   r   close)rI   headersrR   r   DB_PATH
TABLE_NAMEconnr   sanitized_headerscols_sqlrN   col_typecreate_table_sqlcolumn_namesplaceholders
insert_sqldata_to_insert	row_tupler$   r%   )r   r   r   r&   r     sP    
(
 

r   )Fr(   NN)"r   rD   ry   sysr}   r   r   dotenvr   r   	keepa_apir   r   r   r   r   r	   field_mappingsr
   
processingr   stable_calculationsr   r   seller_infor   r   business_calculationsr   r   ra   r'   r   r   r%   r%   r%   r&   <module>   s,   
 s