
    tj@                    ~   d Z ddlmZ ddlZddlZddlZddlZddlmZ  ee	      j                         j                  j                  Zej                  j                  d ee             edz  Zej#                         r"ej                  j                  d ee             ddlmZ d	dZedk(  r e e             y)
a  Snapshot the tvviewers reference tables to local parquet.

Why this exists
---------------
`ReferenceData.from_mysql()` pulls ~6M rows across 12 frames from a remote
server. That is fine once, but `run_pipeline()` calls it on EVERY run, so each
upload paid several minutes of network before a single row was estimated.

The reference data changes daily at most, so it is snapshotted here and read
back from parquet in seconds. `worker/config.json` decides which path is used:

    "reference_mode": "fixtures"   -> read the snapshot (fast, what the worker wants)
    "reference_mode": "mysql"      -> go straight to the server (what this script does)

Run nightly from cron:
    0 4 * * *  cd /var/www/html/crystal/crystal2 && python3 worker/build_cache.py

The snapshot is written to a temporary directory and swapped into place only
after every frame has been fetched, so a failure part-way through leaves the
previous good snapshot untouched and the worker keeps running on it.
    )annotationsN)Pathvendor)ReferenceDatac            	        t         dz  dz  } t        j                  | j                               }t	        |j                  d      xs	 t         dz        }|j                  |j                  dz         }t        j                         }t        d|j                  d       dd	
       t        j                  |      }t        dt        |j                         dt        j                         |z
  ddd	
       d}t        |j                  j                               D ].  \  }}||v rdnd}	t        d|ddt        |      dd|	 d	
       0 |D 
cg c]+  }
|
|j                  vs|j                  |
   j                   r|
- }}
|rt        d| d	
       t        dd	
       |j#                         rt%        j&                  |       |j)                  |       ddlm}m} t1        | ||            D ]=  \  }}|j3                  || dz  d       t        d|ddt        |      ddd	
       ? |dz  }|j#                         rdt        j                  |j                               }d	|d<   d |d!<   |dz  j5                  t        j6                  |d"#             t        d$d	
       nt        d%d	
       |j                  |j                  d&z         }|j#                         rt%        j&                  |       |j#                         r|j9                  |       |j9                  |       |j#                         rt%        j&                  |       	 t%        j:                  |d'(       |j=                  d)       |j?                         D ]  }|j=                  d*        	 |d.z  }|j#                         r|jE                          t        d/d	
       tG        d0 |jI                  d1      D              d2z  }t        d3| d4|d5d6t        j                         |z
  dd7d	
       t        d8       yc c}
w # t@        tB        f$ r*}t        d+| d,| d	
       t        d-d	
       Y d }~d }~ww xY w)9Nworkerzconfig.jsonfixtures_dirreference_cachez	.buildingz[cache] pulling reference from ref_db_hostu    …T)flushz[cache] fetched z frames in z.0fs)ott_multiplierspan_channelspan_mappingz
  (report) z
[cache]   38s z>9,z rowsz0[cache] WARNING: report frames empty or absent: z<[cache]          the OTT, OOH and PAN sheets will read zero.r   )
EPG_FRAMESfetch_epg_referencez.parquetF)indexzcalibration.jsonstalez@fitted against the previous snapshot; re-run worker/fit_model.pystale_reason   )indentz5[cache] carried calibration.json across, marked staleu@   [cache] no calibration.json to carry — run worker/fit_model.pyz	.previouszwww-data)groupi  i  z1[cache] WARNING: could not set group www-data on z: zH[cache]          the worker may not be able to cache its evidence index.zevidence_index.pkluB   [cache] dropped evidence_index.pkl — it rebuilds on the next runc              3  P   K   | ]  }|j                         j                     y w)N)statst_size).0ps     worker/build_cache.py	<genexpr>zmain.<locals>.<genexpr>   s     BAqvvxBs   $&z	*.parqueti   z[cache] wrote u    — z.1fz MB in zs totalzE[cache] set worker/config.json "reference_mode": "fixtures" to use it)%BASEjsonloads	read_textr   get	with_namenametimeprintr   
from_mysqllenframessorteditemsemptyexistsshutilrmtreesave_fixturesengine.epg_adapterr   r   zip
to_parquet
write_textdumpsrenamechownchmoditerdirPermissionErrorLookupErrorunlinksumglob)cfg_pathcfgtargetstagingt0refREPORT_FRAMESr)   dftagnmissingr   r   old_calcalpreviousr    er   sizes                        r!   mainrT   ,   s   h.H
**X'')
*C #''.)Gd5F.FHFv{{[89G	B	+CGGM,B+C4
HPTU

"
"3
'C	S_-[R8LA
NVZ[FM3::++-. Kb"m3l
4*Ac"gc]%u=TJK ( >Q3::%A)<)<  >G > 	@	JRVWLTXY~~gg
 C
$7$<= Fb
g4& 11?
4*Ac"gc]%8EF ))G~~jj**,-G <N	%	%11$**S2KLETRPX\] k 9:Hh}}hNN6hfV:.V! 	AGGEN	 ))E||~RZ^_B[)ABBWLD	N6(%SzR8LG
T\`a	
UVU>t [) fA&A3OW[\X`deefs   0P4AP9 9Q2 Q--Q2__main__)returnint)__doc__
__future__r   r$   r3   sysr*   pathlibr   __file__resolveparentr#   pathinsertstr_vendoris_direngine.referencer   rT   __name__
SystemExit     r!   <module>ri      s   * #   
  H~&&-- 3t9 
 /
>>HHOOAs7|$ *[| z
TV
 rh   